github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 19:31:44 +00:00
Code Issues Projects Releases Wiki Activity

Allow custom client names to be used for cloud controllers

Browse Source 
* This allows a controller to use cloud provider managed RBAC
  when --use-service-account-credentials is set.
* Create ControllerInitFuncConstructor to pass to init funcs to avoid
  future function signature growth.
* Add comments for context around legacy naming of node controllers.
* Add example for setting client names from cloud controller manager.
This commit is contained in:
Nick Turner 2021-06-25 07:33:31 +00:00
parent f6331c74b6
commit a5b47f7dd0
5 changed files with 106 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/cloud-controller-manager/main.go
View File

@ -32,7 +32,7 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cloud-provider"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
@ -68,7 +68,15 @@ func main() {
nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
fss := cliflag.NamedFlagSets{}
nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))
controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper
controllerInitializers["nodeipam"] = app.ControllerInitFuncConstructor{
// "node-controller" is the shared identity of all node controllers, including node, node lifecycle, and node ipam.
// See https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context.
InitContext: app.ControllerInitContext{
ClientName: "node-controller",
},
Constructor: nodeIpamController.StartNodeIpamControllerWrapper,
}
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)

8
cmd/cloud-controller-manager/nodeipamcontroller.go
View File

@ -52,7 +52,7 @@ type nodeIPAMController struct {
nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions
}
func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
func (nodeIpamController *nodeIPAMController) StartNodeIpamControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
allErrors := nodeIpamController.nodeIPAMControllerOptions.Validate()
if len(allErrors) > 0 {
klog.Fatal("NodeIPAM controller values are not properly set.")
@ -60,11 +60,11 @@ func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(com
nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration)
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud)
return startNodeIpamController(initContext, completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud)
}
}
func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
func startNodeIpamController(initContext app.ControllerInitContext, ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
var serviceCIDR *net.IPNet
var secondaryServiceCIDR *net.IPNet
@ -147,7 +147,7 @@ func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, n
nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
ctx.InformerFactory.Core().V1().Nodes(),
cloud,
ctx.ClientBuilder.ClientOrDie("node-controller"),
ctx.ClientBuilder.ClientOrDie(initContext.ClientName),
clusterCIDRs,
serviceCIDR,
secondaryServiceCIDR,

84
staging/src/k8s.io/cloud-provider/app/controllermanager.go
View File

@ -66,7 +66,7 @@ const (
// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
// initFuncConstructor is a map of named controller groups (you can start more than one in an init func) paired to their InitFuncConstructor.
// additionalFlags provides controller specific flags to be included in the complete set of controller manager flags
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command {
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, controllerInitFuncConstructors map[string]ControllerInitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command {
cmd := &cobra.Command{
Use: "cloud-controller-manager",
Long: `The Cloud controller manager is a daemon that embeds
@ -75,7 +75,7 @@ the cloud specific control loops shipped with Kubernetes.`,
verflag.PrintAndExitIfRequested()
cliflag.PrintFlags(cmd.Flags())
c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return err
@ -83,7 +83,7 @@ the cloud specific control loops shipped with Kubernetes.`,
completedConfig := c.Complete()
cloud := cloudInitializer(completedConfig)
controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)
controllerInitializers := ConstructControllerInitializers(controllerInitFuncConstructors, completedConfig, cloud)
if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
@ -102,7 +102,7 @@ the cloud specific control loops shipped with Kubernetes.`,
}
fs := cmd.Flags()
namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
@ -308,11 +308,11 @@ type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprov
type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)
// InitFuncConstructor is used to construct InitFunc
type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc
type InitFuncConstructor func(initcontext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc
// ControllerNames indicate the default controller we are known.
func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string {
ret := sets.StringKeySet(initFuncConstructors)
func ControllerNames(controllerInitFuncConstructors map[string]ControllerInitFuncConstructor) []string {
ret := sets.StringKeySet(controllerInitFuncConstructors)
return ret.List()
}
@ -321,48 +321,80 @@ var ControllersDisabledByDefault = sets.NewString()
// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
func ConstructControllerInitializers(controllerInitFuncConstructors map[string]ControllerInitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
controllers := map[string]InitFunc{}
for name, constructor := range initFuncConstructors {
controllers[name] = constructor(completedConfig, cloud)
for name, constructor := range controllerInitFuncConstructors {
controllers[name] = constructor.Constructor(constructor.InitContext, completedConfig, cloud)
}
return controllers
}
type ControllerInitFuncConstructor struct {
InitContext ControllerInitContext
Constructor InitFuncConstructor
}
type ControllerInitContext struct {
ClientName string
}
// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
func StartCloudNodeControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeController(completedConfig, cloud, ctx.Stop)
return startCloudNodeController(initContext, completedConfig, cloud, ctx.Stop)
}
}
// startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
// StartCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
func StartCloudNodeLifecycleControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop)
return startCloudNodeLifecycleController(initContext, completedConfig, cloud, ctx.Stop)
}
}
// startServiceControllerWrapper is used to take cloud cofig as input and start service controller
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
// StartServiceControllerWrapper is used to take cloud cofig as input and start service controller
func StartServiceControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startServiceController(completedConfig, cloud, ctx.Stop)
return startServiceController(initContext, completedConfig, cloud, ctx.Stop)
}
}
// startRouteControllerWrapper is used to take cloud cofig as input and start route controller
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
// StartRouteControllerWrapper is used to take cloud cofig as input and start route controller
func StartRouteControllerWrapper(initContext ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startRouteController(completedConfig, cloud, ctx.Stop)
return startRouteController(initContext, completedConfig, cloud, ctx.Stop)
}
}
// DefaultInitFuncConstructors is a map of default named controller groups paired with InitFuncConstructor
var DefaultInitFuncConstructors = map[string]InitFuncConstructor{
"cloud-node": StartCloudNodeControllerWrapper,
"cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper,
"service": startServiceControllerWrapper,
"route": startRouteControllerWrapper,
var DefaultInitFuncConstructors = map[string]ControllerInitFuncConstructor{
// The cloud-node controller shares the "node-controller" identity with the cloud-node-lifecycle
// controller for historical reasons. See
// https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context.
"cloud-node": {
InitContext: ControllerInitContext{
ClientName: "node-controller",
},
Constructor: StartCloudNodeControllerWrapper,
},
"cloud-node-lifecycle": {
InitContext: ControllerInitContext{
ClientName: "node-controller",
},
Constructor: StartCloudNodeLifecycleControllerWrapper,
},
"service": {
InitContext: ControllerInitContext{
ClientName: "service-controller",
},
Constructor: StartServiceControllerWrapper,
},
"route": {
InitContext: ControllerInitContext{
ClientName: "route-controller",
},
Constructor: StartRouteControllerWrapper,
},
}
// CreateControllerContext creates a context struct containing references to resources needed by the

16
staging/src/k8s.io/cloud-provider/app/core.go
View File

@ -39,12 +39,12 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
)
func startCloudNodeController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
func startCloudNodeController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
// Start the CloudNodeController
nodeController, err := cloudnodecontroller.NewCloudNodeController(
ctx.SharedInformers.Core().V1().Nodes(),
// cloud node controller uses existing cluster role from node-controller
ctx.ClientBuilder.ClientOrDie("node-controller"),
ctx.ClientBuilder.ClientOrDie(initContext.ClientName),
cloud,
ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration,
)
@ -58,12 +58,12 @@ func startCloudNodeController(ctx *config.CompletedConfig, cloud cloudprovider.I
return nil, true, nil
}
func startCloudNodeLifecycleController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
func startCloudNodeLifecycleController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
// Start the cloudNodeLifecycleController
cloudNodeLifecycleController, err := cloudnodelifecyclecontroller.NewCloudNodeLifecycleController(
ctx.SharedInformers.Core().V1().Nodes(),
// cloud node lifecycle controller uses existing cluster role from node-controller
ctx.ClientBuilder.ClientOrDie("node-controller"),
ctx.ClientBuilder.ClientOrDie(initContext.ClientName),
cloud,
ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
)
@ -77,11 +77,11 @@ func startCloudNodeLifecycleController(ctx *config.CompletedConfig, cloud cloudp
return nil, true, nil
}
func startServiceController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
func startServiceController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
// Start the service controller
serviceController, err := servicecontroller.New(
cloud,
ctx.ClientBuilder.ClientOrDie("service-controller"),
ctx.ClientBuilder.ClientOrDie(initContext.ClientName),
ctx.SharedInformers.Core().V1().Services(),
ctx.SharedInformers.Core().V1().Nodes(),
ctx.ComponentConfig.KubeCloudShared.ClusterName,
@ -98,7 +98,7 @@ func startServiceController(ctx *config.CompletedConfig, cloud cloudprovider.Int
return nil, true, nil
}
func startRouteController(ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
func startRouteController(initContext ControllerInitContext, ctx *config.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
if !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes {
klog.Infof("Will not configure cloud provider routes, --configure-cloud-routes: %v", ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes)
return nil, false, nil
@ -134,7 +134,7 @@ func startRouteController(ctx *config.CompletedConfig, cloud cloudprovider.Inter
routeController := routecontroller.New(
routes,
ctx.ClientBuilder.ClientOrDie("route-controller"),
ctx.ClientBuilder.ClientOrDie(initContext.ClientName),
ctx.SharedInformers.Core().V1().Nodes(),
ctx.ComponentConfig.KubeCloudShared.ClusterName,
clusterCIDRs,

28
staging/src/k8s.io/cloud-provider/sample/basic_main.go
View File

@ -27,7 +27,7 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cloud-provider"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
@ -49,7 +49,7 @@ func main() {
}
fss := cliflag.NamedFlagSets{}
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop)
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers(), fss, wait.NeverStop)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@ -65,6 +65,30 @@ func main() {
}
}
// If custom ClientNames are used, as below, then the controller will not use
// the API server bootstrapped RBAC, and instead will require it to be installed
// separately.
func controllerInitializers() map[string]app.ControllerInitFuncConstructor {
controllerInitializers := app.DefaultInitFuncConstructors
if constructor, ok := controllerInitializers["cloud-node"]; ok {
constructor.InitContext.ClientName = "mycloud-external-cloud-node-controller"
controllerInitializers["cloud-node"] = constructor
}
if constructor, ok := controllerInitializers["cloud-node-lifecycle"]; ok {
constructor.InitContext.ClientName = "mycloud-external-cloud-node-lifecycle-controller"
controllerInitializers["cloud-node-lifecycle"] = constructor
}
if constructor, ok := controllerInitializers["service"]; ok {
constructor.InitContext.ClientName = "mycloud-external-service-controller"
controllerInitializers["service"] = constructor
}
if constructor, ok := controllerInitializers["route"]; ok {
constructor.InitContext.ClientName = "mycloud-external-route-controller"
controllerInitializers["route"] = constructor
}
return controllerInitializers
}
func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider