Merge pull request #98210 from cici37/fix

Fix flag passing in CCM

cmd/cloud-controller-manager/.import-restrictions

@@ -1,6 +1,8 @@
 rules:
   - selectorRegexp: k8s[.]io/kubernetes
     allowedPrefixes:
+      - k8s.io/kubernetes/cmd/kube-controller-manager/app/options
+      - k8s.io/kubernetes/cmd/kube-controller-manager/app/config
       - k8s.io/kubernetes/pkg/api/legacyscheme
       - k8s.io/kubernetes/pkg/api/service
       - k8s.io/kubernetes/pkg/api/v1/pod
@@ -34,4 +36,7 @@ rules:
       - k8s.io/kubernetes/pkg/util/hash
       - k8s.io/kubernetes/pkg/util/node
       - k8s.io/kubernetes/pkg/util/parsers
-      - k8s.io/kubernetes/pkg/util/taints
+      - k8s.io/kubernetes/pkg/util/taints
+      - k8s.io/kubernetes/pkg/proxy/util
+      - k8s.io/kubernetes/pkg/proxy/util/testing
+      - k8s.io/kubernetes/pkg/util/sysctl

cmd/cloud-controller-manager/README.md

@@ -0,0 +1,14 @@
+# cloud-controller-manager/example
+
+This directory provides an example of how to leverage CCM extension mechanism.
+
+## Purpose
+
+Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, an example is included to demonstrate how to leverage CCM extension mechanism to add a controller.
+Please refer to `k8s.io/cloud-provider/sample` if you do not have the requirement of adding/deleting controllers in CCM.
+
+## Things you should NOT do
+
+1. Vendor in `k8s.io/cmd/cloud-controller-manager`.
+2. Directly modify anything under `k8s.io/cmd/cloud-controller-manager` in this repo. 
+3. Make specific cloud provider changes here.

cmd/cloud-controller-manager/main.go

@@ -19,126 +19,89 @@ limitations under the License.
 
 // This file should be written by each cloud provider.
 // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
-// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
 // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
 // The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.
 
 package main
 
 import (
-	"fmt"
 	"math/rand"
-	"net/http"
 	"os"
 	"time"
 
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/options"
-	"k8s.io/component-base/cli/flag"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
-	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/klog/v2"
-	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	// For existing cloud providers, the option to import legacy providers is still available.
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
 
-const (
-	// cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case.
-	cloudProviderName = "SampleCloudProviderName"
-)
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
-	pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
+	ccmOptions, err := options.NewCloudControllerManagerOptions()
-	_ = pflag.CommandLine.Parse(os.Args[1:])
-
-	// this is an example of allow-listing specific controller loops
-	controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
-
-	s, err := options.NewCloudControllerManagerOptions()
 	if err != nil {
 		klog.Fatalf("unable to initialize command options: %v", err)
 	}
-	c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
+
-	if err != nil {
+	controllerInitializers := app.DefaultInitFuncConstructors
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+	// Here is an example to remove the controller which is not needed.
+	// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
+	//delete(controllerInitializers, "cloud-node-lifecycle")
+
+	// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
+	// generate nodeIPAMConfig. Here is an sample code.
+	// If you do not need additional controller, please ignore.
+
+	nodeIpamController := nodeIPAMController{}
+	nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
+	fss := cliflag.NamedFlagSets{}
+	nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))
+	controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper
+
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	// Here is an sample
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
+	// utilflag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := command.Execute(); err != nil {
 		os.Exit(1)
 	}
+}
 
-	cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
+func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
+	// initialize cloud provider with the cloud provider name and config file provided
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
 	if err != nil {
 		klog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}
 	if cloud == nil {
-		klog.Fatalf("cloud provider is nil")
+		klog.Fatalf("Cloud provider is nil")
 	}
 
 	if !cloud.HasClusterID() {
-		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+		if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
 			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
 		} else {
 			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
 		}
 	}
 
-	// Initialize the cloud provider with a reference to the clientBuilder
+	return cloud
-	cloud.Initialize(c.ClientBuilder, make(chan struct{}))
-	// Set the informer on the user cloud object
-	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
-		informerUserCloud.SetInformers(c.SharedInformers)
-	}
-
-	controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
-
-	// Here is an example to remove the controller which is not needed.
-	// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
-	//delete(controllerInitializers, "cloud-node-lifecycle")
-
-	// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
-	// generate nodeIPAMConfig. Here is an sample code. Please pass the right parameter in your code.
-	// If you do not need additional controller, please ignore.
-	nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{
-		ServiceCIDR:          "sample",
-		SecondaryServiceCIDR: "sample",
-		NodeCIDRMaskSize:     11,
-		NodeCIDRMaskSizeIPv4: 11,
-		NodeCIDRMaskSizeIPv6: 111,
-	}
-	controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeIPAMConfig, cloud)
-
-	command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
-
-	// TODO: once we switch everything over to Cobra commands, we can go back to calling
-	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
-	// normalize func and add the go flag set by hand.
-	// Here is an sample
-	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
-	// utilflag.InitFlags()
-	logs.InitLogs()
-	defer logs.FlushLogs()
-
-	// the flags could be set before execute
-	command.Flags().VisitAll(func(flag *pflag.Flag) {
-		if flag.Name == "cloud-provider" {
-			flag.Value.Set("SampleCloudProviderFlagValue")
-			return
-		}
-	})
-	if err := command.Execute(); err != nil {
-		os.Exit(1)
-	}
-}
-
-func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
-	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
-		return startNodeIpamController(ccmconfig, nodeIPAMConfig, ctx, cloud)
-	}
 }

cmd/cloud-controller-manager/nodeipamcontroller.go

@@ -28,10 +28,12 @@ import (
 
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/controller-manager/pkg/features"
 	"k8s.io/klog/v2"
+	nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
 	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
@@ -45,6 +47,23 @@ const (
 	defaultNodeMaskCIDRIPv6 = 64
 )
 
+type nodeIPAMController struct {
+	nodeIPAMControllerConfiguration nodeipamconfig.NodeIPAMControllerConfiguration
+	nodeIPAMControllerOptions       nodeipamcontrolleroptions.NodeIPAMControllerOptions
+}
+
+func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
+	errors := nodeIpamController.nodeIPAMControllerOptions.Validate()
+	if len(errors) > 0 {
+		klog.Fatal("NodeIPAM controller values are not properly set.")
+	}
+	nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration)
+
+	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+		return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud)
+	}
+}
+
 func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
 	var serviceCIDR *net.IPNet
 	var secondaryServiceCIDR *net.IPNet

hack/.staticcheck_failures

@@ -28,4 +28,3 @@ vendor/k8s.io/client-go/discovery
 vendor/k8s.io/client-go/rest
 vendor/k8s.io/client-go/rest/watch
 vendor/k8s.io/client-go/transport
-vendor/k8s.io/cloud-provider/sample

staging/src/k8s.io/cloud-provider/app/controllermanager.go

@@ -28,24 +28,21 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	cacheddiscovery "k8s.io/client-go/discovery/cached"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/metadata"
-	"k8s.io/client-go/metadata/metadatainformer"
-	"k8s.io/client-go/restmapper"
-	cloudprovider "k8s.io/cloud-provider"
-	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
-	"k8s.io/cloud-provider/options"
-	"k8s.io/controller-manager/pkg/clientbuilder"
-	"k8s.io/controller-manager/pkg/informerfactory"
-
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
+	cacheddiscovery "k8s.io/client-go/discovery/cached"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/metadata"
+	"k8s.io/client-go/metadata/metadatainformer"
+	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	cloudprovider "k8s.io/cloud-provider"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	"k8s.io/cloud-provider/options"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/configz"
@@ -53,6 +50,8 @@ import (
 	"k8s.io/component-base/version"
 	"k8s.io/component-base/version/verflag"
 	genericcontrollermanager "k8s.io/controller-manager/app"
+	"k8s.io/controller-manager/pkg/clientbuilder"
+	"k8s.io/controller-manager/pkg/informerfactory"
 	"k8s.io/klog/v2"
 )
 
@@ -64,21 +63,32 @@ const (
 )
 
 // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
-func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, c *cloudcontrollerconfig.Config, controllerInitializers map[string]InitFunc) *cobra.Command {
+// initFuncConstructor is a map of named controller groups (you can start more than one in an init func) paired to their InitFuncConstructor.
-
+// additionalFlags provides controller specific flags to be included in the complete set of controller manager flags
+func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "cloud-controller-manager",
 		Long: `The Cloud controller manager is a daemon that embeds
 the cloud specific control loops shipped with Kubernetes.`,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			verflag.PrintAndExitIfRequested()
 			cliflag.PrintFlags(cmd.Flags())
 
-			if err := Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil {
+			c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
+			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
-				os.Exit(1)
+				return err
 			}
 
+			completedConfig := c.Complete()
+			cloud := cloudInitializer(completedConfig)
+			controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)
+
+			if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				return err
+			}
+			return nil
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
 			for _, arg := range args {
@@ -91,7 +101,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 	}
 
 	fs := cmd.Flags()
-	namedFlagSets := s.Flags(KnownControllers(controllerInitializers), ControllersDisabledByDefault.List())
+	namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
 
@@ -106,6 +116,10 @@ the cloud specific control loops shipped with Kubernetes.`,
 	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)
 	}
+	for _, f := range additionalFlags.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
 	usageFmt := "Usage:\n  %s\n"
 	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
 	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
@@ -122,7 +136,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 }
 
 // Run runs the ExternalCMServer.  This should never exit.
-func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
+func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
 	// To help debugging, immediately log version
 	klog.Infof("Version: %+v", version.Get())
 
@@ -167,7 +181,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st
 		if err != nil {
 			klog.Fatalf("error building controller context: %v", err)
 		}
-		if err := startControllers(controllerContext, c, ctx.Done(), controllerInitializers); err != nil {
+		if err := startControllers(cloud, controllerContext, c, ctx.Done(), controllerInitializers); err != nil {
 			klog.Fatalf("error running controllers: %v", err)
 		}
 	}
@@ -218,7 +232,13 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st
 }
 
 // startControllers starts the cloud specific controller loops.
-func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error {
+func startControllers(cloud cloudprovider.Interface, ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error {
+	// Initialize the cloud provider with a reference to the clientBuilder
+	cloud.Initialize(c.ClientBuilder, stopCh)
+	// Set the informer on the user cloud object
+	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
+		informerUserCloud.SetInformers(c.SharedInformers)
+	}
 	for controllerName, initFn := range controllers {
 		if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
 			klog.Warningf("%q is disabled", controllerName)
@@ -251,59 +271,72 @@ func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudco
 	select {}
 }
 
+// InitCloudFunc is used to initialize cloud
+type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface
+
 // InitFunc is used to launch a particular controller.  It may run additional "should I activate checks".
 // Any error returned will cause the controller process to `Fatal`
 // The bool indicates whether the controller was enabled.
 type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)
 
-// KnownControllers indicate the default controller we are known.
+// InitFuncConstructor is used to construct InitFunc
-func KnownControllers(controllerInitializers map[string]InitFunc) []string {
+type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc
-	ret := sets.StringKeySet(controllerInitializers)
+
+// ControllerNames indicate the default controller we are known.
+func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string {
+	ret := sets.StringKeySet(initFuncConstructors)
 	return ret.List()
 }
 
 // ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
 var ControllersDisabledByDefault = sets.NewString()
 
-// DefaultControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
+// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
 // paired to their InitFunc.  This allows for structured downstream composition and subdivision.
-func DefaultControllerInitializers(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
+func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
 	controllers := map[string]InitFunc{}
-	controllers["cloud-node"] = StartCloudNodeControllerWrapper(completedConfig, cloud)
+	for name, constructor := range initFuncConstructors {
-	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleControllerWrapper(completedConfig, cloud)
+		controllers[name] = constructor(completedConfig, cloud)
-	controllers["service"] = startServiceControllerWrapper(completedConfig, cloud)
+	}
-	controllers["route"] = startRouteControllerWrapper(completedConfig, cloud)
 	return controllers
 }
 
 // StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller
-func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
 	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
 		return startCloudNodeController(completedConfig, cloud, ctx.Stop)
 	}
 }
 
 // startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
-func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
 	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
 		return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop)
 	}
 }
 
 // startServiceControllerWrapper is used to take cloud cofig as input and start service controller
-func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
 	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
 		return startServiceController(completedConfig, cloud, ctx.Stop)
 	}
 }
 
 // startRouteControllerWrapper is used to take cloud cofig as input and start route controller
-func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
 	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
 		return startRouteController(completedConfig, cloud, ctx.Stop)
 	}
 }
 
+// DefaultInitFuncConstructors is a map of default named controller groups paired with InitFuncConstructor
+var DefaultInitFuncConstructors = map[string]InitFuncConstructor{
+	"cloud-node":           StartCloudNodeControllerWrapper,
+	"cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper,
+	"service":              startServiceControllerWrapper,
+	"route":                startRouteControllerWrapper,
+}
+
 // CreateControllerContext creates a context struct containing references to resources needed by the
 // controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
 // the shared-informers client and token controller.

staging/src/k8s.io/cloud-provider/app/testing/testserver.go

@@ -22,17 +22,18 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
 	"k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/options"
-
+	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
 )
 
 // TearDownFunc is to be called to tear down a test server.
@@ -42,7 +43,7 @@ type TearDownFunc func()
 type TestServer struct {
 	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
 	Options              *options.CloudControllerManagerOptions
-	Config               *config.Config
+	Config               *config.CompletedConfig
 	TearDownFn           TearDownFunc // TearDown function
 	TmpDir               string       // Temp Dir used, by the apiserver
 }
@@ -62,6 +63,8 @@ type Logger interface {
 // 		 enough time to remove temporary files.
 func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
 	stopCh := make(chan struct{})
+	configDoneCh := make(chan struct{})
+	var capturedConfig config.CompletedConfig
 	tearDown := func() {
 		close(stopCh)
 		if len(result.TmpDir) != 0 {
@@ -79,58 +82,95 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 		return result, fmt.Errorf("failed to create temp dir: %v", err)
 	}
 
-	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
-
 	s, err := options.NewCloudControllerManagerOptions()
 	if err != nil {
 		return TestServer{}, err
 	}
-	namedFlagSets := s.Flags([]string{}, []string{})
+
-	for _, f := range namedFlagSets.FlagSets {
+	cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface {
-		fs.AddFlagSet(f)
+		capturedConfig = *config
-	}
+		// send signal to indicate the capturedConfig has been properly set
-	fs.Parse(customFlags)
+		close(configDoneCh)
-	if s.SecureServing.BindPort != 0 {
+		cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
-		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
+		cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
 		if err != nil {
-			return result, fmt.Errorf("failed to create listener: %v", err)
+			t.Fatalf("Cloud provider could not be initialized: %v", err)
 		}
 		s.SecureServing.ServerCert.CertDirectory = result.TmpDir
+		if cloud == nil {
+			t.Fatalf("Cloud provider is nil")
+		}
+		return cloud
+	}
+	fss := cliflag.NamedFlagSets{}
+	command := app.NewCloudControllerManagerCommand(s, cloudInitializer, app.DefaultInitFuncConstructors, fss, stopCh)
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 
-		t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
+	commandArgs := []string{}
+	listeners := []net.Listener{}
+	disableInsecure := false
+	disableSecure := false
+	for _, arg := range customFlags {
+		if strings.HasPrefix(arg, "--secure-port=") {
+			if arg == "--secure-port=0" {
+				commandArgs = append(commandArgs, arg)
+				disableSecure = true
+			}
+		} else if strings.HasPrefix(arg, "--port=") {
+			if arg == "--port=0" {
+				commandArgs = append(commandArgs, arg)
+				disableInsecure = true
+			}
+		} else if strings.HasPrefix(arg, "--cert-dir=") {
+			// skip it
+		} else {
+			commandArgs = append(commandArgs, arg)
+		}
 	}
 
-	if s.InsecureServing.BindPort != 0 {
+	if !disableSecure {
-		s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
+		listener, bindPort, err := createListenerOnFreePort()
 		if err != nil {
 			return result, fmt.Errorf("failed to create listener: %v", err)
 		}
+		listeners = append(listeners, listener)
+		commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort))
+		commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir))
 
-		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
+		t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
 	}
+	if !disableInsecure {
+		listener, bindPort, err := createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+		listeners = append(listeners, listener)
+		commandArgs = append(commandArgs, fmt.Sprintf("--port=%d", bindPort))
 
-	config, err := s.Config([]string{}, []string{})
+		t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
-	if err != nil {
-		return result, fmt.Errorf("failed to create config from options: %v", err)
 	}
-	cloudConfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider
+	for _, listener := range listeners {
-	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
+		listener.Close()
-	if err != nil {
-		return result, fmt.Errorf("cloud provider could not be initialized: %v", err)
-	}
-	if cloud == nil {
-		return result, fmt.Errorf("cloud provider is nil")
 	}
 
 	errCh := make(chan error)
-	go func(stopCh <-chan struct{}) {
+	go func() {
-		if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
+		command.SetArgs(commandArgs)
+		if err := command.Execute(); err != nil {
 			errCh <- err
 		}
-	}(stopCh)
+		close(errCh)
+	}()
+
+	select {
+	case <-configDoneCh:
+
+	case err := <-errCh:
+		return result, err
+	}
 
 	t.Logf("Waiting for /healthz to be ok...")
-	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
+	client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig)
 	if err != nil {
 		return result, fmt.Errorf("failed to create a client: %v", err)
 	}
@@ -154,9 +194,9 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 	}
 
 	// from here the caller must call tearDown
-	result.LoopbackClientConfig = config.LoopbackClientConfig
+	result.LoopbackClientConfig = capturedConfig.LoopbackClientConfig
 	result.Options = s
-	result.Config = config
+	result.Config = &capturedConfig
 	result.TearDownFn = tearDown
 
 	return result, nil

staging/src/k8s.io/cloud-provider/sample/README.md

@@ -4,13 +4,13 @@ This directory provides sample code about how all cloud providers should leverag
 
 ## Purpose
 
-Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. 
+Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. 
 
-## Steps cloud providers shoud follow
+## Steps cloud providers should follow
 
 1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-<provider>`
-2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minial working sample and `advanced_main.go` for advanced configuration samples.
+2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample.
-Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for detailed samples.
+Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for extra details.
 3. Build/release CCM from your external repo. For existing cloud providers, the option to import legacy providers from `k8s.io/legacy-cloud-provider/<provider>` is still available.
 
 ## Things you should NOT do

staging/src/k8s.io/cloud-provider/sample/advanced_main.go

@@ -1,115 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// This file should be written by each cloud provider.
-// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
-// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
-// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
-
-package sample
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"time"
-
-	"github.com/spf13/pflag"
-
-	"k8s.io/cloud-provider"
-	"k8s.io/cloud-provider/app"
-	"k8s.io/cloud-provider/options"
-	"k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/logs"
-	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
-	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
-	"k8s.io/klog/v2"
-	// For existing cloud providers, the option to import legacy providers is still available.
-	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
-)
-
-const (
-	// The variables below are samples, please edit the value for your case.
-
-	// cloudProviderName shows an sample of using hard coded parameter
-	cloudProviderName = "SampleCloudProviderName"
-)
-
-func advancedMain() {
-	rand.Seed(time.Now().UnixNano())
-
-	pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
-	_ = pflag.CommandLine.Parse(os.Args[1:])
-
-	// this is an example of allow-listing specific controller loops
-	controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
-
-	s, err := options.NewCloudControllerManagerOptions()
-	if err != nil {
-		klog.Fatalf("unable to initialize command options: %v", err)
-	}
-	c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-
-	cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
-	if err != nil {
-		klog.Fatalf("Cloud provider could not be initialized: %v", err)
-	}
-	if cloud == nil {
-		klog.Fatalf("cloud provider is nil")
-	}
-
-	if !cloud.HasClusterID() {
-		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
-			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
-		} else {
-			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
-		}
-	}
-
-	// Initialize the cloud provider with a reference to the clientBuilder
-	cloud.Initialize(c.ClientBuilder, make(chan struct{}))
-	// Set the informer on the user cloud object
-	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
-		informerUserCloud.SetInformers(c.SharedInformers)
-	}
-
-	controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
-	command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
-
-	// TODO: once we switch everything over to Cobra commands, we can go back to calling
-	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
-	// normalize func and add the go flag set by hand.
-	// Here is an sample
-	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
-	// utilflag.InitFlags()
-	logs.InitLogs()
-	defer logs.FlushLogs()
-
-	// the flags could be set before execute
-	command.Flags().VisitAll(func(flag *pflag.Flag) {
-		if flag.Name == "cloud-provider" {
-			flag.Value.Set("SampleCloudProviderFlagValue")
-			return
-		}
-	})
-	if err := command.Execute(); err != nil {
-		os.Exit(1)
-	}
-}

staging/src/k8s.io/cloud-provider/sample/basic_main.go

@@ -16,23 +16,22 @@ limitations under the License.
 
 // This file should be written by each cloud provider.
 // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
-// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
 // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
 
-package sample
+package main
 
 import (
-	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/spf13/pflag"
-
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
+	"k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/options"
-	"k8s.io/component-base/cli/flag"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
@@ -41,70 +40,49 @@ import (
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
 
-const (
-	// The variables below are samples, please edit the value for your case.
-
-	// sampleCloudProviderName shows an sample of using hard coded parameter for CloudProviderName
-	sampleCloudProviderName = "SampleCloudProviderName"
-)
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
-	s, err := options.NewCloudControllerManagerOptions()
+	ccmOptions, err := options.NewCloudControllerManagerOptions()
 	if err != nil {
 		klog.Fatalf("unable to initialize command options: %v", err)
 	}
-	c, err := s.Config([]string{}, app.ControllersDisabledByDefault.List())
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
 
-	// initialize cloud provider with the cloud provider name and config file provided
+	fss := cliflag.NamedFlagSets{}
-	cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop)
-	if err != nil {
-		klog.Fatalf("Cloud provider could not be initialized: %v", err)
-	}
-	if cloud == nil {
-		klog.Fatalf("cloud provider is nil")
-	}
-
-	if !cloud.HasClusterID() {
-		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
-			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
-		} else {
-			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
-		}
-	}
-
-	// Initialize the cloud provider with a reference to the clientBuilder
-	cloud.Initialize(c.ClientBuilder, make(chan struct{}))
-	// Set the informer on the user cloud object
-	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
-		informerUserCloud.SetInformers(c.SharedInformers)
-	}
-
-	controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
-	command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
 	// normalize func and add the go flag set by hand.
 	// Here is an sample
-	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	// the flags could be set before execute
-	command.Flags().VisitAll(func(flag *pflag.Flag) {
-		if flag.Name == "cloud-provider" {
-			flag.Value.Set("SampleCloudProviderFlagValue")
-			return
-		}
-	})
 	if err := command.Execute(); err != nil {
 		os.Exit(1)
 	}
 }
+
+func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
+
+	// initialize cloud provider with the cloud provider name and config file provided
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
+	if err != nil {
+		klog.Fatalf("Cloud provider could not be initialized: %v", err)
+	}
+	if cloud == nil {
+		klog.Fatalf("Cloud provider is nil")
+	}
+
+	if !cloud.HasClusterID() {
+		if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+	return cloud
+}