Fix flag passing in CCM.

This commit is contained in:
cici37 2021-01-14 13:07:53 -08:00
parent 35c233f18d
commit 2b3d2303a5
7 changed files with 111 additions and 227 deletions

View File

@ -19,14 +19,12 @@ limitations under the License.
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
// The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.
package main
import (
"fmt"
"math/rand"
"net/http"
"os"
@ -57,47 +55,41 @@ const (
func main() {
rand.Seed(time.Now().UnixNano())
pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
_ = pflag.CommandLine.Parse(os.Args[1:])
// this is an example of allow-listing specific controller loops
controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
s, err := options.NewCloudControllerManagerOptions()
ccmOptions, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
cloudInitializer := func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, cloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("Cloud provider is nil")
}
if !cloud.HasClusterID() {
if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(config.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(config.SharedInformers)
}
return cloud
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
controllerInitializers := app.DefaultInitFuncConstructors
// Here is an example to remove the controller which is not needed.
// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
//delete(controllerInitializers, "cloud-node-lifecycle")
@ -105,16 +97,9 @@ func main() {
// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
// generate nodeIPAMConfig. Here is an sample code. Please pass the right parameter in your code.
// If you do not need additional controller, please ignore.
nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{
ServiceCIDR: "sample",
SecondaryServiceCIDR: "sample",
NodeCIDRMaskSize: 11,
NodeCIDRMaskSizeIPv4: 11,
NodeCIDRMaskSizeIPv6: 111,
}
controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeIPAMConfig, cloud)
controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
command := app.NewCloudControllerManagerCommand(cloudProviderName, ccmOptions, cloudInitializer, controllerInitializers)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@ -125,20 +110,20 @@ func main() {
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}
func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{
ServiceCIDR: "sample",
SecondaryServiceCIDR: "sample",
NodeCIDRMaskSize: 11,
NodeCIDRMaskSizeIPv4: 11,
NodeCIDRMaskSizeIPv6: 111,
}
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startNodeIpamController(ccmconfig, nodeIPAMConfig, ctx, cloud)
return startNodeIpamController(completedConfig, nodeIPAMConfig, ctx, cloud)
}
}

View File

@ -64,7 +64,7 @@ const (
)
// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, c *cloudcontrollerconfig.Config, controllerInitializers map[string]InitFunc) *cobra.Command {
func NewCloudControllerManagerCommand(cloudProviderName string, s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor) *cobra.Command {
cmd := &cobra.Command{
Use: "cloud-controller-manager",
@ -72,13 +72,27 @@ func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions,
the cloud specific control loops shipped with Kubernetes.`,
Run: func(cmd *cobra.Command, args []string) {
verflag.PrintAndExitIfRequested()
cloudProviderFlag := cmd.Flags().Lookup("cloud-provider")
if cloudProviderFlag.Value.String() == "" {
cloudProviderFlag.Value.Set(cloudProviderName)
}
cliflag.PrintFlags(cmd.Flags())
if err := Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil {
c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
completedConfig := c.Complete()
cloud := cloudInitializer(completedConfig)
controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)
if err := Run(completedConfig, controllerInitializers, wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
Args: func(cmd *cobra.Command, args []string) error {
for _, arg := range args {
@ -91,7 +105,7 @@ the cloud specific control loops shipped with Kubernetes.`,
}
fs := cmd.Flags()
namedFlagSets := s.Flags(KnownControllers(controllerInitializers), ControllersDisabledByDefault.List())
namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
@ -121,6 +135,8 @@ the cloud specific control loops shipped with Kubernetes.`,
return cmd
}
type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface
// Run runs the ExternalCMServer. This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
// To help debugging, immediately log version
@ -256,54 +272,62 @@ func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudco
// The bool indicates whether the controller was enabled.
type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)
// KnownControllers indicate the default controller we are known.
func KnownControllers(controllerInitializers map[string]InitFunc) []string {
ret := sets.StringKeySet(controllerInitializers)
type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc
// ControllerNames indicate the default controller we are known.
func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string {
ret := sets.StringKeySet(initFuncConstructors)
return ret.List()
}
// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
var ControllersDisabledByDefault = sets.NewString()
// DefaultControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func DefaultControllerInitializers(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["cloud-node"] = StartCloudNodeControllerWrapper(completedConfig, cloud)
controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleControllerWrapper(completedConfig, cloud)
controllers["service"] = startServiceControllerWrapper(completedConfig, cloud)
controllers["route"] = startRouteControllerWrapper(completedConfig, cloud)
for name, constructor := range initFuncConstructors {
controllers[name] = constructor(completedConfig, cloud)
}
return controllers
}
// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeController(completedConfig, cloud, ctx.Stop)
}
}
// startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop)
}
}
// startServiceControllerWrapper is used to take cloud cofig as input and start service controller
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startServiceController(completedConfig, cloud, ctx.Stop)
}
}
// startRouteControllerWrapper is used to take cloud cofig as input and start route controller
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startRouteController(completedConfig, cloud, ctx.Stop)
}
}
var DefaultInitFuncConstructors = map[string]InitFuncConstructor{
"cloud-node": StartCloudNodeControllerWrapper,
"cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper,
"service": startServiceControllerWrapper,
"route": startRouteControllerWrapper,
}
// CreateControllerContext creates a context struct containing references to resources needed by the
// controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
// the shared-informers client and token controller.

View File

@ -124,7 +124,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
errCh := make(chan error)
go func(stopCh <-chan struct{}) {
if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
if err := app.Run(config.Complete(), app.ConstructControllerInitializers(app.DefaultInitFuncConstructors, config.Complete(), cloud), stopCh); err != nil {
errCh <- err
}
}(stopCh)

View File

@ -9,8 +9,8 @@ Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/k
## Steps cloud providers shoud follow
1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-<provider>`
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minial working sample and `advanced_main.go` for advanced configuration samples.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for detailed samples.
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for extra details.
3. Build/release CCM from your external repo. For existing cloud providers, the option to import legacy providers from `k8s.io/legacy-cloud-provider/<provider>` is still available.
## Things you should NOT do

View File

@ -1,115 +0,0 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
package sample
import (
"fmt"
"math/rand"
"os"
"time"
"github.com/spf13/pflag"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
"k8s.io/klog/v2"
// For existing cloud providers, the option to import legacy providers is still available.
// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
)
const (
// The variables below are samples, please edit the value for your case.
// cloudProviderName shows an sample of using hard coded parameter
cloudProviderName = "SampleCloudProviderName"
)
func advancedMain() {
rand.Seed(time.Now().UnixNano())
pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
_ = pflag.CommandLine.Parse(os.Args[1:])
// this is an example of allow-listing specific controller loops
controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
s, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
// normalize func and add the go flag set by hand.
// Here is an sample
pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
// utilflag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}

View File

@ -16,21 +16,19 @@ limitations under the License.
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
package sample
import (
"fmt"
"math/rand"
"os"
"time"
"github.com/spf13/pflag"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
@ -51,42 +49,41 @@ const (
func main() {
rand.Seed(time.Now().UnixNano())
s, err := options.NewCloudControllerManagerOptions()
ccmOptions, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config([]string{}, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface {
cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, cloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("Cloud provider is nil")
}
if !cloud.HasClusterID() {
if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(config.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(config.SharedInformers)
}
return cloud
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
command := app.NewCloudControllerManagerCommand(sampleCloudProviderName, ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@ -97,13 +94,6 @@ func main() {
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}