diff --git a/cmd/cloud-controller-manager/.import-restrictions b/cmd/cloud-controller-manager/.import-restrictions index 4515e2f7be5..151e8744804 100644 --- a/cmd/cloud-controller-manager/.import-restrictions +++ b/cmd/cloud-controller-manager/.import-restrictions @@ -1,6 +1,8 @@ rules: - selectorRegexp: k8s[.]io/kubernetes allowedPrefixes: + - k8s.io/kubernetes/cmd/kube-controller-manager/app/options + - k8s.io/kubernetes/cmd/kube-controller-manager/app/config - k8s.io/kubernetes/pkg/api/legacyscheme - k8s.io/kubernetes/pkg/api/service - k8s.io/kubernetes/pkg/api/v1/pod @@ -34,4 +36,7 @@ rules: - k8s.io/kubernetes/pkg/util/hash - k8s.io/kubernetes/pkg/util/node - k8s.io/kubernetes/pkg/util/parsers - - k8s.io/kubernetes/pkg/util/taints \ No newline at end of file + - k8s.io/kubernetes/pkg/util/taints + - k8s.io/kubernetes/pkg/proxy/util + - k8s.io/kubernetes/pkg/proxy/util/testing + - k8s.io/kubernetes/pkg/util/sysctl \ No newline at end of file diff --git a/cmd/cloud-controller-manager/README.md b/cmd/cloud-controller-manager/README.md new file mode 100644 index 00000000000..7248ce39e47 --- /dev/null +++ b/cmd/cloud-controller-manager/README.md @@ -0,0 +1,14 @@ +# cloud-controller-manager/example + +This directory provides an example of how to leverage CCM extension mechanism. + +## Purpose + +Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, an example is included to demonstrate how to leverage CCM extension mechanism to add a controller. +Please refer to `k8s.io/cloud-provider/sample` if you do not have the requirement of adding/deleting controllers in CCM. + +## Things you should NOT do + +1. Vendor in `k8s.io/cmd/cloud-controller-manager`. +2. Directly modify anything under `k8s.io/cmd/cloud-controller-manager` in this repo. +3. Make specific cloud provider changes here. diff --git a/cmd/cloud-controller-manager/main.go b/cmd/cloud-controller-manager/main.go index 5629c22a092..23ec3eb65f6 100644 --- a/cmd/cloud-controller-manager/main.go +++ b/cmd/cloud-controller-manager/main.go @@ -19,126 +19,89 @@ limitations under the License. // This file should be written by each cloud provider. // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go -// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go // The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use. package main import ( - "fmt" "math/rand" - "net/http" "os" "time" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" - "k8s.io/component-base/cli/flag" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/logs" _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - genericcontrollermanager "k8s.io/controller-manager/app" "k8s.io/klog/v2" - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" // For existing cloud providers, the option to import legacy providers is still available. // e.g. _"k8s.io/legacy-cloud-providers/" ) -const ( - // cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case. - cloudProviderName = "SampleCloudProviderName" -) - func main() { rand.Seed(time.Now().UnixNano()) - pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true - _ = pflag.CommandLine.Parse(os.Args[1:]) - - // this is an example of allow-listing specific controller loops - controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"} - - s, err := options.NewCloudControllerManagerOptions() + ccmOptions, err := options.NewCloudControllerManagerOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) } - c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + + controllerInitializers := app.DefaultInitFuncConstructors + // Here is an example to remove the controller which is not needed. + // e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need. + //delete(controllerInitializers, "cloud-node-lifecycle") + + // Here is an example to add an controller(NodeIpamController) which will be used by cloud provider + // generate nodeIPAMConfig. Here is an sample code. + // If you do not need additional controller, please ignore. + + nodeIpamController := nodeIPAMController{} + nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration + fss := cliflag.NamedFlagSets{} + nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller")) + controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper + + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) + + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + // Here is an sample + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + if err := command.Execute(); err != nil { os.Exit(1) } +} - cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) +func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface { + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + // initialize cloud provider with the cloud provider name and config file provided + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { klog.Fatalf("Cloud provider could not be initialized: %v", err) } if cloud == nil { - klog.Fatalf("cloud provider is nil") + klog.Fatalf("Cloud provider is nil") } if !cloud.HasClusterID() { - if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { + if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") } else { klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") } } - // Initialize the cloud provider with a reference to the clientBuilder - cloud.Initialize(c.ClientBuilder, make(chan struct{})) - // Set the informer on the user cloud object - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(c.SharedInformers) - } - - controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud) - - // Here is an example to remove the controller which is not needed. - // e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need. - //delete(controllerInitializers, "cloud-node-lifecycle") - - // Here is an example to add an controller(NodeIpamController) which will be used by cloud provider - // generate nodeIPAMConfig. Here is an sample code. Please pass the right parameter in your code. - // If you do not need additional controller, please ignore. - nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{ - ServiceCIDR: "sample", - SecondaryServiceCIDR: "sample", - NodeCIDRMaskSize: 11, - NodeCIDRMaskSizeIPv4: 11, - NodeCIDRMaskSizeIPv6: 111, - } - controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeIPAMConfig, cloud) - - command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers) - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - // Here is an sample - pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - // the flags could be set before execute - command.Flags().VisitAll(func(flag *pflag.Flag) { - if flag.Name == "cloud-provider" { - flag.Value.Set("SampleCloudProviderFlagValue") - return - } - }) - if err := command.Execute(); err != nil { - os.Exit(1) - } -} - -func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startNodeIpamController(ccmconfig, nodeIPAMConfig, ctx, cloud) - } + return cloud } diff --git a/cmd/cloud-controller-manager/nodeipamcontroller.go b/cmd/cloud-controller-manager/nodeipamcontroller.go index b96fa6c80c7..ebceef0629f 100644 --- a/cmd/cloud-controller-manager/nodeipamcontroller.go +++ b/cmd/cloud-controller-manager/nodeipamcontroller.go @@ -28,10 +28,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" "k8s.io/controller-manager/pkg/features" "k8s.io/klog/v2" + nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam" nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" @@ -45,6 +47,23 @@ const ( defaultNodeMaskCIDRIPv6 = 64 ) +type nodeIPAMController struct { + nodeIPAMControllerConfiguration nodeipamconfig.NodeIPAMControllerConfiguration + nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions +} + +func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { + errors := nodeIpamController.nodeIPAMControllerOptions.Validate() + if len(errors) > 0 { + klog.Fatal("NodeIPAM controller values are not properly set.") + } + nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration) + + return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { + return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud) + } +} + func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) { var serviceCIDR *net.IPNet var secondaryServiceCIDR *net.IPNet diff --git a/hack/.staticcheck_failures b/hack/.staticcheck_failures index 22d7e8b0f6f..a3bcf3c6439 100644 --- a/hack/.staticcheck_failures +++ b/hack/.staticcheck_failures @@ -28,4 +28,3 @@ vendor/k8s.io/client-go/discovery vendor/k8s.io/client-go/rest vendor/k8s.io/client-go/rest/watch vendor/k8s.io/client-go/transport -vendor/k8s.io/cloud-provider/sample diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go index 7c7e9a90061..4acefc386a8 100644 --- a/staging/src/k8s.io/cloud-provider/app/controllermanager.go +++ b/staging/src/k8s.io/cloud-provider/app/controllermanager.go @@ -28,24 +28,21 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - cacheddiscovery "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/informers" - "k8s.io/client-go/metadata" - "k8s.io/client-go/metadata/metadatainformer" - "k8s.io/client-go/restmapper" - cloudprovider "k8s.io/cloud-provider" - cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - "k8s.io/cloud-provider/options" - "k8s.io/controller-manager/pkg/clientbuilder" - "k8s.io/controller-manager/pkg/informerfactory" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" + cacheddiscovery "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/informers" + "k8s.io/client-go/metadata" + "k8s.io/client-go/metadata/metadatainformer" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" + cloudprovider "k8s.io/cloud-provider" + cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/configz" @@ -53,6 +50,8 @@ import ( "k8s.io/component-base/version" "k8s.io/component-base/version/verflag" genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/controller-manager/pkg/clientbuilder" + "k8s.io/controller-manager/pkg/informerfactory" "k8s.io/klog/v2" ) @@ -64,21 +63,32 @@ const ( ) // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters -func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, c *cloudcontrollerconfig.Config, controllerInitializers map[string]InitFunc) *cobra.Command { - +// initFuncConstructor is a map of named controller groups (you can start more than one in an init func) paired to their InitFuncConstructor. +// additionalFlags provides controller specific flags to be included in the complete set of controller manager flags +func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command { cmd := &cobra.Command{ Use: "cloud-controller-manager", Long: `The Cloud controller manager is a daemon that embeds the cloud specific control loops shipped with Kubernetes.`, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { verflag.PrintAndExitIfRequested() cliflag.PrintFlags(cmd.Flags()) - if err := Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil { + c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List()) + if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) + return err } + completedConfig := c.Complete() + cloud := cloudInitializer(completedConfig) + controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud) + + if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil }, Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { @@ -91,7 +101,7 @@ the cloud specific control loops shipped with Kubernetes.`, } fs := cmd.Flags() - namedFlagSets := s.Flags(KnownControllers(controllerInitializers), ControllersDisabledByDefault.List()) + namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List()) verflag.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) @@ -106,6 +116,10 @@ the cloud specific control loops shipped with Kubernetes.`, for _, f := range namedFlagSets.FlagSets { fs.AddFlagSet(f) } + for _, f := range additionalFlags.FlagSets { + fs.AddFlagSet(f) + } + usageFmt := "Usage:\n %s\n" cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) cmd.SetUsageFunc(func(cmd *cobra.Command) error { @@ -122,7 +136,7 @@ the cloud specific control loops shipped with Kubernetes.`, } // Run runs the ExternalCMServer. This should never exit. -func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error { +func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error { // To help debugging, immediately log version klog.Infof("Version: %+v", version.Get()) @@ -167,7 +181,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st if err != nil { klog.Fatalf("error building controller context: %v", err) } - if err := startControllers(controllerContext, c, ctx.Done(), controllerInitializers); err != nil { + if err := startControllers(cloud, controllerContext, c, ctx.Done(), controllerInitializers); err != nil { klog.Fatalf("error running controllers: %v", err) } } @@ -218,7 +232,13 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st } // startControllers starts the cloud specific controller loops. -func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error { +func startControllers(cloud cloudprovider.Interface, ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error { + // Initialize the cloud provider with a reference to the clientBuilder + cloud.Initialize(c.ClientBuilder, stopCh) + // Set the informer on the user cloud object + if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { + informerUserCloud.SetInformers(c.SharedInformers) + } for controllerName, initFn := range controllers { if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) { klog.Warningf("%q is disabled", controllerName) @@ -251,59 +271,72 @@ func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudco select {} } +// InitCloudFunc is used to initialize cloud +type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface + // InitFunc is used to launch a particular controller. It may run additional "should I activate checks". // Any error returned will cause the controller process to `Fatal` // The bool indicates whether the controller was enabled. type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error) -// KnownControllers indicate the default controller we are known. -func KnownControllers(controllerInitializers map[string]InitFunc) []string { - ret := sets.StringKeySet(controllerInitializers) +// InitFuncConstructor is used to construct InitFunc +type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc + +// ControllerNames indicate the default controller we are known. +func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string { + ret := sets.StringKeySet(initFuncConstructors) return ret.List() } // ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. var ControllersDisabledByDefault = sets.NewString() -// DefaultControllerInitializers is a private map of named controller groups (you can start more than one in an init func) +// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func) // paired to their InitFunc. This allows for structured downstream composition and subdivision. -func DefaultControllerInitializers(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc { +func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc { controllers := map[string]InitFunc{} - controllers["cloud-node"] = StartCloudNodeControllerWrapper(completedConfig, cloud) - controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleControllerWrapper(completedConfig, cloud) - controllers["service"] = startServiceControllerWrapper(completedConfig, cloud) - controllers["route"] = startRouteControllerWrapper(completedConfig, cloud) + for name, constructor := range initFuncConstructors { + controllers[name] = constructor(completedConfig, cloud) + } return controllers } // StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller -func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { +func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { return startCloudNodeController(completedConfig, cloud, ctx.Stop) } } // startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller -func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { +func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop) } } // startServiceControllerWrapper is used to take cloud cofig as input and start service controller -func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { +func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { return startServiceController(completedConfig, cloud, ctx.Stop) } } // startRouteControllerWrapper is used to take cloud cofig as input and start route controller -func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { +func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc { return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { return startRouteController(completedConfig, cloud, ctx.Stop) } } +// DefaultInitFuncConstructors is a map of default named controller groups paired with InitFuncConstructor +var DefaultInitFuncConstructors = map[string]InitFuncConstructor{ + "cloud-node": StartCloudNodeControllerWrapper, + "cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper, + "service": startServiceControllerWrapper, + "route": startRouteControllerWrapper, +} + // CreateControllerContext creates a context struct containing references to resources needed by the // controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for // the shared-informers client and token controller. diff --git a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go index aecd90a8ec9..9e334bf5524 100644 --- a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go +++ b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go @@ -22,17 +22,18 @@ import ( "io/ioutil" "net" "os" + "strings" "time" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" + cliflag "k8s.io/component-base/cli/flag" ) // TearDownFunc is to be called to tear down a test server. @@ -42,7 +43,7 @@ type TearDownFunc func() type TestServer struct { LoopbackClientConfig *restclient.Config // Rest client config using the magic token Options *options.CloudControllerManagerOptions - Config *config.Config + Config *config.CompletedConfig TearDownFn TearDownFunc // TearDown function TmpDir string // Temp Dir used, by the apiserver } @@ -62,6 +63,8 @@ type Logger interface { // enough time to remove temporary files. func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { stopCh := make(chan struct{}) + configDoneCh := make(chan struct{}) + var capturedConfig config.CompletedConfig tearDown := func() { close(stopCh) if len(result.TmpDir) != 0 { @@ -79,58 +82,95 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err return result, fmt.Errorf("failed to create temp dir: %v", err) } - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s, err := options.NewCloudControllerManagerOptions() if err != nil { return TestServer{}, err } - namedFlagSets := s.Flags([]string{}, []string{}) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - fs.Parse(customFlags) - if s.SecureServing.BindPort != 0 { - s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + + cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface { + capturedConfig = *config + // send signal to indicate the capturedConfig has been properly set + close(configDoneCh) + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) + t.Fatalf("Cloud provider could not be initialized: %v", err) } s.SecureServing.ServerCert.CertDirectory = result.TmpDir + if cloud == nil { + t.Fatalf("Cloud provider is nil") + } + return cloud + } + fss := cliflag.NamedFlagSets{} + command := app.NewCloudControllerManagerCommand(s, cloudInitializer, app.DefaultInitFuncConstructors, fss, stopCh) + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + commandArgs := []string{} + listeners := []net.Listener{} + disableInsecure := false + disableSecure := false + for _, arg := range customFlags { + if strings.HasPrefix(arg, "--secure-port=") { + if arg == "--secure-port=0" { + commandArgs = append(commandArgs, arg) + disableSecure = true + } + } else if strings.HasPrefix(arg, "--port=") { + if arg == "--port=0" { + commandArgs = append(commandArgs, arg) + disableInsecure = true + } + } else if strings.HasPrefix(arg, "--cert-dir=") { + // skip it + } else { + commandArgs = append(commandArgs, arg) + } } - if s.InsecureServing.BindPort != 0 { - s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() + if !disableSecure { + listener, bindPort, err := createListenerOnFreePort() if err != nil { return result, fmt.Errorf("failed to create listener: %v", err) } + listeners = append(listeners, listener) + commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir)) - t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) + t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) } + if !disableInsecure { + listener, bindPort, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + listeners = append(listeners, listener) + commandArgs = append(commandArgs, fmt.Sprintf("--port=%d", bindPort)) - config, err := s.Config([]string{}, []string{}) - if err != nil { - return result, fmt.Errorf("failed to create config from options: %v", err) + t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) } - cloudConfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider - cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) - if err != nil { - return result, fmt.Errorf("cloud provider could not be initialized: %v", err) - } - if cloud == nil { - return result, fmt.Errorf("cloud provider is nil") + for _, listener := range listeners { + listener.Close() } errCh := make(chan error) - go func(stopCh <-chan struct{}) { - if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil { + go func() { + command.SetArgs(commandArgs) + if err := command.Execute(); err != nil { errCh <- err } - }(stopCh) + close(errCh) + }() + + select { + case <-configDoneCh: + + case err := <-errCh: + return result, err + } t.Logf("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) } @@ -154,9 +194,9 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // from here the caller must call tearDown - result.LoopbackClientConfig = config.LoopbackClientConfig + result.LoopbackClientConfig = capturedConfig.LoopbackClientConfig result.Options = s - result.Config = config + result.Config = &capturedConfig result.TearDownFn = tearDown return result, nil diff --git a/staging/src/k8s.io/cloud-provider/sample/README.md b/staging/src/k8s.io/cloud-provider/sample/README.md index 04ce9d0da5f..369ee82ba55 100644 --- a/staging/src/k8s.io/cloud-provider/sample/README.md +++ b/staging/src/k8s.io/cloud-provider/sample/README.md @@ -4,13 +4,13 @@ This directory provides sample code about how all cloud providers should leverag ## Purpose -Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. +Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. -## Steps cloud providers shoud follow +## Steps cloud providers should follow 1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-` -2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minial working sample and `advanced_main.go` for advanced configuration samples. -Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for detailed samples. +2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample. +Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for extra details. 3. Build/release CCM from your external repo. For existing cloud providers, the option to import legacy providers from `k8s.io/legacy-cloud-provider/` is still available. ## Things you should NOT do diff --git a/staging/src/k8s.io/cloud-provider/sample/advanced_main.go b/staging/src/k8s.io/cloud-provider/sample/advanced_main.go deleted file mode 100644 index b541ba531f3..00000000000 --- a/staging/src/k8s.io/cloud-provider/sample/advanced_main.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file should be written by each cloud provider. -// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go -// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go -// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go - -package sample - -import ( - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - - "k8s.io/cloud-provider" - "k8s.io/cloud-provider/app" - "k8s.io/cloud-provider/options" - "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/klog/v2" - // For existing cloud providers, the option to import legacy providers is still available. - // e.g. _"k8s.io/legacy-cloud-providers/" -) - -const ( - // The variables below are samples, please edit the value for your case. - - // cloudProviderName shows an sample of using hard coded parameter - cloudProviderName = "SampleCloudProviderName" -) - -func advancedMain() { - rand.Seed(time.Now().UnixNano()) - - pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true - _ = pflag.CommandLine.Parse(os.Args[1:]) - - // this is an example of allow-listing specific controller loops - controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"} - - s, err := options.NewCloudControllerManagerOptions() - if err != nil { - klog.Fatalf("unable to initialize command options: %v", err) - } - c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) - if err != nil { - klog.Fatalf("Cloud provider could not be initialized: %v", err) - } - if cloud == nil { - klog.Fatalf("cloud provider is nil") - } - - if !cloud.HasClusterID() { - if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { - klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") - } else { - klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") - } - } - - // Initialize the cloud provider with a reference to the clientBuilder - cloud.Initialize(c.ClientBuilder, make(chan struct{})) - // Set the informer on the user cloud object - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(c.SharedInformers) - } - - controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud) - command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers) - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - // Here is an sample - pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - // the flags could be set before execute - command.Flags().VisitAll(func(flag *pflag.Flag) { - if flag.Name == "cloud-provider" { - flag.Value.Set("SampleCloudProviderFlagValue") - return - } - }) - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/staging/src/k8s.io/cloud-provider/sample/basic_main.go b/staging/src/k8s.io/cloud-provider/sample/basic_main.go index 73df23faeb6..9f6386c5430 100644 --- a/staging/src/k8s.io/cloud-provider/sample/basic_main.go +++ b/staging/src/k8s.io/cloud-provider/sample/basic_main.go @@ -16,23 +16,22 @@ limitations under the License. // This file should be written by each cloud provider. // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go -// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go -package sample +package main import ( - "fmt" "math/rand" "os" "time" "github.com/spf13/pflag" - + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" + "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" - "k8s.io/component-base/cli/flag" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/logs" _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration @@ -41,70 +40,49 @@ import ( // e.g. _"k8s.io/legacy-cloud-providers/" ) -const ( - // The variables below are samples, please edit the value for your case. - - // sampleCloudProviderName shows an sample of using hard coded parameter for CloudProviderName - sampleCloudProviderName = "SampleCloudProviderName" -) - func main() { rand.Seed(time.Now().UnixNano()) - s, err := options.NewCloudControllerManagerOptions() + ccmOptions, err := options.NewCloudControllerManagerOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) } - c, err := s.Config([]string{}, app.ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - // initialize cloud provider with the cloud provider name and config file provided - cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) - if err != nil { - klog.Fatalf("Cloud provider could not be initialized: %v", err) - } - if cloud == nil { - klog.Fatalf("cloud provider is nil") - } - - if !cloud.HasClusterID() { - if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { - klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") - } else { - klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") - } - } - - // Initialize the cloud provider with a reference to the clientBuilder - cloud.Initialize(c.ClientBuilder, make(chan struct{})) - // Set the informer on the user cloud object - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(c.SharedInformers) - } - - controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud) - command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers) + fss := cliflag.NamedFlagSets{} + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop) // TODO: once we switch everything over to Cobra commands, we can go back to calling // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the // normalize func and add the go flag set by hand. // Here is an sample - pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) // utilflag.InitFlags() logs.InitLogs() defer logs.FlushLogs() - // the flags could be set before execute - command.Flags().VisitAll(func(flag *pflag.Flag) { - if flag.Name == "cloud-provider" { - flag.Value.Set("SampleCloudProviderFlagValue") - return - } - }) if err := command.Execute(); err != nil { os.Exit(1) } } + +func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + + // initialize cloud provider with the cloud provider name and config file provided + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) + if err != nil { + klog.Fatalf("Cloud provider could not be initialized: %v", err) + } + if cloud == nil { + klog.Fatalf("Cloud provider is nil") + } + + if !cloud.HasClusterID() { + if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { + klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") + } else { + klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") + } + } + return cloud +}