github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #98210 from cici37/fix

Browse Source 
Fix flag passing in CCM
This commit is contained in:
Kubernetes Prow Robot 2021-03-03 14:40:01 -08:00 committed by GitHub
commit e5538fa94f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 253 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/cloud-controller-manager/.import-restrictions
View File

@ -1,6 +1,8 @@
rules:
- selectorRegexp: k8s[.]io/kubernetes
allowedPrefixes:
- k8s.io/kubernetes/cmd/kube-controller-manager/app/options
- k8s.io/kubernetes/cmd/kube-controller-manager/app/config
- k8s.io/kubernetes/pkg/api/legacyscheme
- k8s.io/kubernetes/pkg/api/service
- k8s.io/kubernetes/pkg/api/v1/pod
@ -35,3 +37,6 @@ rules:
- k8s.io/kubernetes/pkg/util/node
- k8s.io/kubernetes/pkg/util/parsers
- k8s.io/kubernetes/pkg/util/taints
- k8s.io/kubernetes/pkg/proxy/util
- k8s.io/kubernetes/pkg/proxy/util/testing
- k8s.io/kubernetes/pkg/util/sysctl

14
cmd/cloud-controller-manager/README.md Normal file
View File

@ -0,0 +1,14 @@
# cloud-controller-manager/example
This directory provides an example of how to leverage CCM extension mechanism.
## Purpose
Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, an example is included to demonstrate how to leverage CCM extension mechanism to add a controller.
Please refer to `k8s.io/cloud-provider/sample` if you do not have the requirement of adding/deleting controllers in CCM.
## Things you should NOT do
1. Vendor in `k8s.io/cmd/cloud-controller-manager`.
2. Directly modify anything under `k8s.io/cmd/cloud-controller-manager` in this repo.
3. Make specific cloud provider changes here.

115
cmd/cloud-controller-manager/main.go
View File

@ -19,126 +19,89 @@ limitations under the License.
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
// The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.
package main
import (
"fmt"
"math/rand"
"net/http"
"os"
"time"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli/flag"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
genericcontrollermanager "k8s.io/controller-manager/app"
"k8s.io/klog/v2"
nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
// For existing cloud providers, the option to import legacy providers is still available.
// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
)
const (
// cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case.
cloudProviderName = "SampleCloudProviderName"
)
func main() {
rand.Seed(time.Now().UnixNano())
pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
_ = pflag.CommandLine.Parse(os.Args[1:])
// this is an example of allow-listing specific controller loops
controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
s, err := options.NewCloudControllerManagerOptions()
ccmOptions, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
controllerInitializers := app.DefaultInitFuncConstructors
// Here is an example to remove the controller which is not needed.
// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
//delete(controllerInitializers, "cloud-node-lifecycle")
// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
// generate nodeIPAMConfig. Here is an sample code.
// If you do not need additional controller, please ignore.
nodeIpamController := nodeIPAMController{}
nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
fss := cliflag.NamedFlagSets{}
nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))
controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
// normalize func and add the go flag set by hand.
// Here is an sample
pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
// utilflag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
if err := command.Execute(); err != nil {
os.Exit(1)
}
}
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
klog.Fatalf("Cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
// Here is an example to remove the controller which is not needed.
// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
//delete(controllerInitializers, "cloud-node-lifecycle")
// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
// generate nodeIPAMConfig. Here is an sample code. Please pass the right parameter in your code.
// If you do not need additional controller, please ignore.
nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{
ServiceCIDR: "sample",
SecondaryServiceCIDR: "sample",
NodeCIDRMaskSize: 11,
NodeCIDRMaskSizeIPv4: 11,
NodeCIDRMaskSizeIPv6: 111,
}
controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeIPAMConfig, cloud)
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
// normalize func and add the go flag set by hand.
// Here is an sample
pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
// utilflag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}
func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startNodeIpamController(ccmconfig, nodeIPAMConfig, ctx, cloud)
}
return cloud
}

19
cmd/cloud-controller-manager/nodeipamcontroller.go
View File

@ -28,10 +28,12 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
genericcontrollermanager "k8s.io/controller-manager/app"
"k8s.io/controller-manager/pkg/features"
"k8s.io/klog/v2"
nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
@ -45,6 +47,23 @@ const (
defaultNodeMaskCIDRIPv6 = 64
)
type nodeIPAMController struct {
nodeIPAMControllerConfiguration nodeipamconfig.NodeIPAMControllerConfiguration
nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions
}
func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
errors := nodeIpamController.nodeIPAMControllerOptions.Validate()
if len(errors) > 0 {
klog.Fatal("NodeIPAM controller values are not properly set.")
}
nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration)
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud)
}
}
func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
var serviceCIDR *net.IPNet
var secondaryServiceCIDR *net.IPNet

1
hack/.staticcheck_failures
View File

@ -28,4 +28,3 @@ vendor/k8s.io/client-go/discovery
vendor/k8s.io/client-go/rest
vendor/k8s.io/client-go/rest/watch
vendor/k8s.io/client-go/transport
vendor/k8s.io/cloud-provider/sample

99
staging/src/k8s.io/cloud-provider/app/controllermanager.go
View File

@ -28,24 +28,21 @@ import (
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
cacheddiscovery "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/informers"
"k8s.io/client-go/metadata"
"k8s.io/client-go/metadata/metadatainformer"
"k8s.io/client-go/restmapper"
cloudprovider "k8s.io/cloud-provider"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
"k8s.io/controller-manager/pkg/clientbuilder"
"k8s.io/controller-manager/pkg/informerfactory"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
cacheddiscovery "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/informers"
"k8s.io/client-go/metadata"
"k8s.io/client-go/metadata/metadatainformer"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
cloudprovider "k8s.io/cloud-provider"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
"k8s.io/component-base/configz"
@ -53,6 +50,8 @@ import (
"k8s.io/component-base/version"
"k8s.io/component-base/version/verflag"
genericcontrollermanager "k8s.io/controller-manager/app"
"k8s.io/controller-manager/pkg/clientbuilder"
"k8s.io/controller-manager/pkg/informerfactory"
"k8s.io/klog/v2"
)
@ -64,21 +63,32 @@ const (
)
// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, c *cloudcontrollerconfig.Config, controllerInitializers map[string]InitFunc) *cobra.Command {
// initFuncConstructor is a map of named controller groups (you can start more than one in an init func) paired to their InitFuncConstructor.
// additionalFlags provides controller specific flags to be included in the complete set of controller manager flags
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, additionalFlags cliflag.NamedFlagSets, stopCh <-chan struct{}) *cobra.Command {
cmd := &cobra.Command{
Use: "cloud-controller-manager",
Long: `The Cloud controller manager is a daemon that embeds
the cloud specific control loops shipped with Kubernetes.`,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
cliflag.PrintFlags(cmd.Flags())
if err := Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil {
c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
return err
}
completedConfig := c.Complete()
cloud := cloudInitializer(completedConfig)
controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)
if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return err
}
return nil
},
Args: func(cmd *cobra.Command, args []string) error {
for _, arg := range args {
@ -91,7 +101,7 @@ the cloud specific control loops shipped with Kubernetes.`,
}
fs := cmd.Flags()
namedFlagSets := s.Flags(KnownControllers(controllerInitializers), ControllersDisabledByDefault.List())
namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
@ -106,6 +116,10 @@ the cloud specific control loops shipped with Kubernetes.`,
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
for _, f := range additionalFlags.FlagSets {
fs.AddFlagSet(f)
}
usageFmt := "Usage:\n %s\n"
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
cmd.SetUsageFunc(func(cmd *cobra.Command) error {
@ -122,7 +136,7 @@ the cloud specific control loops shipped with Kubernetes.`,
}
// Run runs the ExternalCMServer. This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
// To help debugging, immediately log version
klog.Infof("Version: %+v", version.Get())
@ -167,7 +181,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st
if err != nil {
klog.Fatalf("error building controller context: %v", err)
}
if err := startControllers(controllerContext, c, ctx.Done(), controllerInitializers); err != nil {
if err := startControllers(cloud, controllerContext, c, ctx.Done(), controllerInitializers); err != nil {
klog.Fatalf("error running controllers: %v", err)
}
}
@ -218,7 +232,13 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[st
}
// startControllers starts the cloud specific controller loops.
func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error {
func startControllers(cloud cloudprovider.Interface, ctx genericcontrollermanager.ControllerContext, c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, controllers map[string]InitFunc) error {
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, stopCh)
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
for controllerName, initFn := range controllers {
if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
klog.Warningf("%q is disabled", controllerName)
@ -251,59 +271,72 @@ func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudco
select {}
}
// InitCloudFunc is used to initialize cloud
type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface
// InitFunc is used to launch a particular controller. It may run additional "should I activate checks".
// Any error returned will cause the controller process to `Fatal`
// The bool indicates whether the controller was enabled.
type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)
// KnownControllers indicate the default controller we are known.
func KnownControllers(controllerInitializers map[string]InitFunc) []string {
ret := sets.StringKeySet(controllerInitializers)
// InitFuncConstructor is used to construct InitFunc
type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc
// ControllerNames indicate the default controller we are known.
func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string {
ret := sets.StringKeySet(initFuncConstructors)
return ret.List()
}
// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
var ControllersDisabledByDefault = sets.NewString()
// DefaultControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func DefaultControllerInitializers(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["cloud-node"] = StartCloudNodeControllerWrapper(completedConfig, cloud)
controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleControllerWrapper(completedConfig, cloud)
controllers["service"] = startServiceControllerWrapper(completedConfig, cloud)
controllers["route"] = startRouteControllerWrapper(completedConfig, cloud)
for name, constructor := range initFuncConstructors {
controllers[name] = constructor(completedConfig, cloud)
}
return controllers
}
// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeController(completedConfig, cloud, ctx.Stop)
}
}
// startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop)
}
}
// startServiceControllerWrapper is used to take cloud cofig as input and start service controller
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startServiceController(completedConfig, cloud, ctx.Stop)
}
}
// startRouteControllerWrapper is used to take cloud cofig as input and start route controller
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startRouteController(completedConfig, cloud, ctx.Stop)
}
}
// DefaultInitFuncConstructors is a map of default named controller groups paired with InitFuncConstructor
var DefaultInitFuncConstructors = map[string]InitFuncConstructor{
"cloud-node": StartCloudNodeControllerWrapper,
"cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper,
"service": startServiceControllerWrapper,
"route": startRouteControllerWrapper,
}
// CreateControllerContext creates a context struct containing references to resources needed by the
// controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
// the shared-informers client and token controller.

126
staging/src/k8s.io/cloud-provider/app/testing/testserver.go
View File

@ -22,17 +22,18 @@ import (
"io/ioutil"
"net"
"os"
"strings"
"time"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
)
// TearDownFunc is to be called to tear down a test server.
@ -42,7 +43,7 @@ type TearDownFunc func()
type TestServer struct {
LoopbackClientConfig *restclient.Config // Rest client config using the magic token
Options *options.CloudControllerManagerOptions
Config *config.Config
Config *config.CompletedConfig
TearDownFn TearDownFunc // TearDown function
TmpDir string // Temp Dir used, by the apiserver
}
@ -62,6 +63,8 @@ type Logger interface {
// enough time to remove temporary files.
func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
stopCh := make(chan struct{})
configDoneCh := make(chan struct{})
var capturedConfig config.CompletedConfig
tearDown := func() {
close(stopCh)
if len(result.TmpDir) != 0 {
@ -79,58 +82,95 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
return result, fmt.Errorf("failed to create temp dir: %v", err)
}
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
s, err := options.NewCloudControllerManagerOptions()
if err != nil {
return TestServer{}, err
}
namedFlagSets := s.Flags([]string{}, []string{})
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
fs.Parse(customFlags)
if s.SecureServing.BindPort != 0 {
s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return result, fmt.Errorf("failed to create listener: %v", err)
}
s.SecureServing.ServerCert.CertDirectory = result.TmpDir
t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
}
if s.InsecureServing.BindPort != 0 {
s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return result, fmt.Errorf("failed to create listener: %v", err)
}
t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
}
config, err := s.Config([]string{}, []string{})
if err != nil {
return result, fmt.Errorf("failed to create config from options: %v", err)
}
cloudConfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider
cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface {
capturedConfig = *config
// send signal to indicate the capturedConfig has been properly set
close(configDoneCh)
cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
if err != nil {
return result, fmt.Errorf("cloud provider could not be initialized: %v", err)
t.Fatalf("Cloud provider could not be initialized: %v", err)
}
s.SecureServing.ServerCert.CertDirectory = result.TmpDir
if cloud == nil {
return result, fmt.Errorf("cloud provider is nil")
t.Fatalf("Cloud provider is nil")
}
return cloud
}
fss := cliflag.NamedFlagSets{}
command := app.NewCloudControllerManagerCommand(s, cloudInitializer, app.DefaultInitFuncConstructors, fss, stopCh)
pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
commandArgs := []string{}
listeners := []net.Listener{}
disableInsecure := false
disableSecure := false
for _, arg := range customFlags {
if strings.HasPrefix(arg, "--secure-port=") {
if arg == "--secure-port=0" {
commandArgs = append(commandArgs, arg)
disableSecure = true
}
} else if strings.HasPrefix(arg, "--port=") {
if arg == "--port=0" {
commandArgs = append(commandArgs, arg)
disableInsecure = true
}
} else if strings.HasPrefix(arg, "--cert-dir=") {
// skip it
} else {
commandArgs = append(commandArgs, arg)
}
}
if !disableSecure {
listener, bindPort, err := createListenerOnFreePort()
if err != nil {
return result, fmt.Errorf("failed to create listener: %v", err)
}
listeners = append(listeners, listener)
commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort))
commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir))
t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
}
if !disableInsecure {
listener, bindPort, err := createListenerOnFreePort()
if err != nil {
return result, fmt.Errorf("failed to create listener: %v", err)
}
listeners = append(listeners, listener)
commandArgs = append(commandArgs, fmt.Sprintf("--port=%d", bindPort))
t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
}
for _, listener := range listeners {
listener.Close()
}
errCh := make(chan error)
go func(stopCh <-chan struct{}) {
if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
go func() {
command.SetArgs(commandArgs)
if err := command.Execute(); err != nil {
errCh <- err
}
}(stopCh)
close(errCh)
}()
select {
case <-configDoneCh:
case err := <-errCh:
return result, err
}
t.Logf("Waiting for /healthz to be ok...")
client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig)
if err != nil {
return result, fmt.Errorf("failed to create a client: %v", err)
}
@ -154,9 +194,9 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
}
// from here the caller must call tearDown
result.LoopbackClientConfig = config.LoopbackClientConfig
result.LoopbackClientConfig = capturedConfig.LoopbackClientConfig
result.Options = s
result.Config = config
result.Config = &capturedConfig
result.TearDownFn = tearDown
return result, nil

8
staging/src/k8s.io/cloud-provider/sample/README.md
View File

@ -4,13 +4,13 @@ This directory provides sample code about how all cloud providers should leverag
## Purpose
Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager.
Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager.
## Steps cloud providers shoud follow
## Steps cloud providers should follow
1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-<provider>`
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minial working sample and `advanced_main.go` for advanced configuration samples.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for detailed samples.
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for extra details.
3. Build/release CCM from your external repo. For existing cloud providers, the option to import legacy providers from `k8s.io/legacy-cloud-provider/<provider>` is still available.
## Things you should NOT do

115
staging/src/k8s.io/cloud-provider/sample/advanced_main.go
View File

@ -1,115 +0,0 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
package sample
import (
"fmt"
"math/rand"
"os"
"time"
"github.com/spf13/pflag"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
"k8s.io/klog/v2"
// For existing cloud providers, the option to import legacy providers is still available.
// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
)
const (
// The variables below are samples, please edit the value for your case.
// cloudProviderName shows an sample of using hard coded parameter
cloudProviderName = "SampleCloudProviderName"
)
func advancedMain() {
rand.Seed(time.Now().UnixNano())
pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
_ = pflag.CommandLine.Parse(os.Args[1:])
// this is an example of allow-listing specific controller loops
controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
s, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
// normalize func and add the go flag set by hand.
// Here is an sample
pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
// utilflag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}

82
staging/src/k8s.io/cloud-provider/sample/basic_main.go
View File

@ -16,23 +16,22 @@ limitations under the License.
// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
package sample
package main
import (
"fmt"
"math/rand"
"os"
"time"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli/flag"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
@ -41,70 +40,49 @@ import (
// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
)
const (
// The variables below are samples, please edit the value for your case.
// sampleCloudProviderName shows an sample of using hard coded parameter for CloudProviderName
sampleCloudProviderName = "SampleCloudProviderName"
)
func main() {
rand.Seed(time.Now().UnixNano())
s, err := options.NewCloudControllerManagerOptions()
ccmOptions, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config([]string{}, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
fss := cliflag.NamedFlagSets{}
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop)
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
// normalize func and add the go flag set by hand.
// Here is an sample
pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
// utilflag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}
func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("Cloud provider is nil")
}
if !cloud.HasClusterID() {
if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
return cloud
}