From 8aaa527d35ab9d1805c252a839725e3a586cef71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 2 Sep 2018 14:10:46 +0300 Subject: [PATCH] Fixup cmd/*controller-manager code after struct changes. Co-authored by @stewart-yu --- .../app/controllermanager.go | 24 ++--- .../app/options/options.go | 72 ++++++-------- .../app/options/options_test.go | 63 ++++++------ cmd/controller-manager/app/options/generic.go | 83 +++++++++++----- .../app/options/kubecloudshared.go | 27 +++-- .../app/controllermanager.go | 24 ++--- cmd/kube-controller-manager/app/core.go | 10 +- .../app/options/endpointcontroller.go | 12 +-- .../app/options/nodeipamcontroller.go | 12 +-- .../app/options/options.go | 99 ++++++------------- .../app/options/options_test.go | 39 ++++---- 11 files changed, 234 insertions(+), 231 deletions(-) diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 6599099248b..3f615147230 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -102,7 +102,7 @@ the cloud specific control loops shipped with Kubernetes.`, // Run runs the ExternalCMServer. This should never exit. func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error { - cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.CloudProvider.Name, c.ComponentConfig.CloudProvider.CloudConfigFile) + cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } @@ -127,14 +127,14 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error // Start the controller manager HTTP server if c.SecureServing != nil { - unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } if c.InsecureServing != nil { - unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { @@ -148,7 +148,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error } } - if !c.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { + if !c.ComponentConfig.Generic.LeaderElection.LeaderElect { run(context.TODO()) panic("unreachable") } @@ -162,7 +162,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error id = id + "_" + string(uuid.NewUUID()) // Lock required for leader election - rl, err := resourcelock.New(c.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, + rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock, "kube-system", "cloud-controller-manager", c.LeaderElectionClient.CoreV1(), @@ -177,9 +177,9 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error // Try and become the leader and start cloud controller manager loops leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ Lock: rl, - LeaseDuration: c.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, - RenewDeadline: c.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, - RetryPeriod: c.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, + LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration, + RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration, + RetryPeriod: c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { @@ -208,12 +208,12 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru c.ComponentConfig.NodeStatusUpdateFrequency.Duration) nodeController.Run(stop) - time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) // Start the PersistentVolumeLabelController pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud) go pvlController.Run(5, stop) - time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) // Start the service controller serviceController, err := servicecontroller.New( @@ -227,7 +227,7 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru glog.Errorf("Failed to start service controller: %v", err) } else { go serviceController.Run(stop, int(c.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) - time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) } // If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller @@ -245,7 +245,7 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru routeController := routecontroller.New(routes, client("route-controller"), c.SharedInformers.Core().V1().Nodes(), c.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDR) go routeController.Run(stop, c.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) - time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) } } else { glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 90f298d54b3..8abfad41d30 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -55,9 +55,7 @@ const ( // CloudControllerManagerOptions is the main context object for the controller manager. type CloudControllerManagerOptions struct { - CloudProvider *cmoptions.CloudProviderOptions - Debugging *cmoptions.DebuggingOptions - GenericComponent *cmoptions.GenericComponentConfigOptions + Generic *cmoptions.GenericControllerManagerConfigurationOptions KubeCloudShared *cmoptions.KubeCloudSharedOptions ServiceController *cmoptions.ServiceControllerOptions @@ -82,17 +80,15 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) } s := CloudControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{}, - Debugging: &cmoptions.DebuggingOptions{}, - GenericComponent: cmoptions.NewGenericComponentConfigOptions(componentConfig.GenericComponent), - KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), + Generic: cmoptions.NewGenericControllerManagerConfigurationOptions(componentConfig.Generic), + KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, }, SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), - BindPort: int(componentConfig.KubeCloudShared.Port), + BindAddress: net.ParseIP(componentConfig.Generic.Address), + BindPort: int(componentConfig.Generic.Port), BindNetwork: "tcp", }).WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), @@ -112,31 +108,35 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) } // NewDefaultComponentConfig returns cloud-controller manager configuration object. -func NewDefaultComponentConfig(insecurePort int32) (componentconfig.CloudControllerManagerConfiguration, error) { +func NewDefaultComponentConfig(insecurePort int32) (*componentconfig.CloudControllerManagerConfiguration, error) { + // TODO: This code will be fixed up/improved when the ccm API types are moved to their own, real API group out of + // pkg/apis/componentconfig to cmd/cloud-controller-manager/app/apis/ scheme := runtime.NewScheme() if err := componentconfigv1alpha1.AddToScheme(scheme); err != nil { - return componentconfig.CloudControllerManagerConfiguration{}, err + return nil, err } if err := componentconfig.AddToScheme(scheme); err != nil { - return componentconfig.CloudControllerManagerConfiguration{}, err + return nil, err } + scheme.AddKnownTypes(componentconfigv1alpha1.SchemeGroupVersion, &componentconfigv1alpha1.CloudControllerManagerConfiguration{}) + scheme.AddKnownTypes(componentconfig.SchemeGroupVersion, &componentconfig.CloudControllerManagerConfiguration{}) - versioned := componentconfigv1alpha1.CloudControllerManagerConfiguration{} - scheme.Default(&versioned) - - internal := componentconfig.CloudControllerManagerConfiguration{} - if err := scheme.Convert(&versioned, &internal, nil); err != nil { + versioned := &componentconfigv1alpha1.CloudControllerManagerConfiguration{} + internal := &componentconfig.CloudControllerManagerConfiguration{} + scheme.Default(versioned) + if err := scheme.Convert(versioned, internal, nil); err != nil { return internal, err } - internal.KubeCloudShared.Port = insecurePort + internal.Generic.Port = insecurePort return internal, nil } // Flags returns flags for a specific APIServer by section name -func (o *CloudControllerManagerOptions) Flags() (fss apiserverflag.NamedFlagSets) { - o.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) - o.Debugging.AddFlags(fss.FlagSet("debugging")) - o.GenericComponent.AddFlags(fss.FlagSet("generic")) +func (o *CloudControllerManagerOptions) Flags() apiserverflag.NamedFlagSets { + fss := apiserverflag.NamedFlagSets{} + o.Generic.AddFlags(&fss, []string{}, []string{}) + // TODO: Implement the --controllers flag fully for the ccm + fss.FlagSet("generic").MarkHidden("controllers") o.KubeCloudShared.AddFlags(fss.FlagSet("generic")) o.ServiceController.AddFlags(fss.FlagSet("service controller")) @@ -158,13 +158,7 @@ func (o *CloudControllerManagerOptions) Flags() (fss apiserverflag.NamedFlagSets // ApplyTo fills up cloud controller manager config with options. func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, userAgent string) error { var err error - if err = o.CloudProvider.ApplyTo(&c.ComponentConfig.CloudProvider); err != nil { - return err - } - if err = o.Debugging.ApplyTo(&c.ComponentConfig.Debugging); err != nil { - return err - } - if err = o.GenericComponent.ApplyTo(&c.ComponentConfig.GenericComponent); err != nil { + if err = o.Generic.ApplyTo(&c.ComponentConfig.Generic); err != nil { return err } if err = o.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { @@ -192,9 +186,9 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, if err != nil { return err } - c.Kubeconfig.ContentConfig.ContentType = o.GenericComponent.ContentType - c.Kubeconfig.QPS = o.GenericComponent.KubeAPIQPS - c.Kubeconfig.Burst = int(o.GenericComponent.KubeAPIBurst) + c.Kubeconfig.ContentConfig.ContentType = o.Generic.ClientConnection.ContentType + c.Kubeconfig.QPS = o.Generic.ClientConnection.QPS + c.Kubeconfig.Burst = int(o.Generic.ClientConnection.Burst) c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent)) if err != nil { @@ -213,7 +207,7 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), CoreClient: c.Client.CoreV1(), AuthenticationClient: c.Client.AuthenticationV1(), - Namespace: "kube-system", + Namespace: metav1.NamespaceSystem, } } else { c.ClientBuilder = rootClientBuilder @@ -223,8 +217,8 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, // sync back to component config // TODO: find more elegant way than syncing back the values. - c.ComponentConfig.KubeCloudShared.Port = int32(o.InsecureServing.BindPort) - c.ComponentConfig.KubeCloudShared.Address = o.InsecureServing.BindAddress.String() + c.ComponentConfig.Generic.Port = int32(o.InsecureServing.BindPort) + c.ComponentConfig.Generic.Address = o.InsecureServing.BindAddress.String() c.ComponentConfig.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency @@ -235,9 +229,7 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, func (o *CloudControllerManagerOptions) Validate() error { errors := []error{} - errors = append(errors, o.CloudProvider.Validate()...) - errors = append(errors, o.Debugging.Validate()...) - errors = append(errors, o.GenericComponent.Validate()...) + errors = append(errors, o.Generic.Validate(nil, nil)...) errors = append(errors, o.KubeCloudShared.Validate()...) errors = append(errors, o.ServiceController.Validate()...) errors = append(errors, o.SecureServing.Validate()...) @@ -245,7 +237,7 @@ func (o *CloudControllerManagerOptions) Validate() error { errors = append(errors, o.Authentication.Validate()...) errors = append(errors, o.Authorization.Validate()...) - if len(o.CloudProvider.Name) == 0 { + if len(o.KubeCloudShared.CloudProvider.Name) == 0 { errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty")) } @@ -256,7 +248,7 @@ func (o *CloudControllerManagerOptions) Validate() error { func resyncPeriod(c *cloudcontrollerconfig.Config) func() time.Duration { return func() time.Duration { factor := rand.Float64() + 1 - return time.Duration(float64(c.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) + return time.Duration(float64(c.ComponentConfig.Generic.MinResyncPeriod.Nanoseconds()) * factor) } } diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index 9467f634fd6..f0c23807416 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/pflag" + apimachineryconfig "k8s.io/apimachinery/pkg/apis/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" apiserverconfig "k8s.io/apiserver/pkg/apis/config" @@ -35,18 +36,15 @@ func TestDefaultFlags(t *testing.T) { s, _ := NewCloudControllerManagerOptions() expected := &CloudControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "", - CloudConfigFile: "", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableContentionProfiling: false, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 20.0, - KubeAPIBurst: 30, + Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ + Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, + ClientConnection: apimachineryconfig.ClientConnectionConfiguration{ + ContentType: "application/vnd.kubernetes.protobuf", + QPS: 20.0, + Burst: 30, + }, ControllerStartInterval: metav1.Duration{Duration: 0}, LeaderElection: apiserverconfig.LeaderElectionConfiguration{ ResourceLock: "endpoints", @@ -55,10 +53,12 @@ func TestDefaultFlags(t *testing.T) { RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + Debugging: &cmoptions.DebuggingOptions{ + EnableContentionProfiling: false, + }, + Controllers: []string{"*"}, }, KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10253, // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, ClusterName: "kubernetes", @@ -66,6 +66,10 @@ func TestDefaultFlags(t *testing.T) { AllocateNodeCIDRs: false, CIDRAllocatorType: "", ConfigureCloudRoutes: true, + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "", + CloudConfigFile: "", + }, }, ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: 1, @@ -150,18 +154,15 @@ func TestAddFlags(t *testing.T) { fs.Parse(args) expected := &CloudControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableContentionProfiling: true, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, + Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ + Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, + ClientConnection: apimachineryconfig.ClientConnectionConfiguration{ + ContentType: "application/vnd.kubernetes.protobuf", + QPS: 50.0, + Burst: 100, + }, ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, LeaderElection: apiserverconfig.LeaderElectionConfiguration{ ResourceLock: "configmap", @@ -170,10 +171,16 @@ func TestAddFlags(t *testing.T) { RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, }, + Debugging: &cmoptions.DebuggingOptions{ + EnableContentionProfiling: true, + }, + Controllers: []string{"*"}, }, KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10253, // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "gce", + CloudConfigFile: "/cloud-config", + }, RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, ClusterName: "k8s", diff --git a/cmd/controller-manager/app/options/generic.go b/cmd/controller-manager/app/options/generic.go index 66523abb730..61c4c41f89c 100644 --- a/cmd/controller-manager/app/options/generic.go +++ b/cmd/controller-manager/app/options/generic.go @@ -17,77 +17,112 @@ limitations under the License. package options import ( - "github.com/spf13/pflag" + "fmt" + "strings" + apimachineryconfig "k8s.io/apimachinery/pkg/apis/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" apiserverconfig "k8s.io/apiserver/pkg/apis/config" + apiserverflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/leaderelectionconfig" ) -// GenericComponentConfigOptions holds the options which are generic. -type GenericComponentConfigOptions struct { +// GenericControllerManagerConfigurationOptions holds the options which are generic. +type GenericControllerManagerConfigurationOptions struct { + Port int32 + Address string MinResyncPeriod metav1.Duration - ContentType string - KubeAPIQPS float32 - KubeAPIBurst int32 + ClientConnection apimachineryconfig.ClientConnectionConfiguration ControllerStartInterval metav1.Duration LeaderElection apiserverconfig.LeaderElectionConfiguration + Debugging *DebuggingOptions + Controllers []string } -// NewGenericComponentConfigOptions returns generic configuration default values for both +// NewGenericControllerManagerConfigurationOptions returns generic configuration default values for both // the kube-controller-manager and the cloud-contoller-manager. Any common changes should // be made here. Any individual changes should be made in that controller. -func NewGenericComponentConfigOptions(cfg componentconfig.GenericComponentConfiguration) *GenericComponentConfigOptions { - o := &GenericComponentConfigOptions{ +func NewGenericControllerManagerConfigurationOptions(cfg componentconfig.GenericControllerManagerConfiguration) *GenericControllerManagerConfigurationOptions { + o := &GenericControllerManagerConfigurationOptions{ + Port: cfg.Port, + Address: cfg.Address, MinResyncPeriod: cfg.MinResyncPeriod, - ContentType: cfg.ContentType, - KubeAPIQPS: cfg.KubeAPIQPS, - KubeAPIBurst: cfg.KubeAPIBurst, + ClientConnection: cfg.ClientConnection, ControllerStartInterval: cfg.ControllerStartInterval, LeaderElection: cfg.LeaderElection, + Debugging: &DebuggingOptions{}, + Controllers: cfg.Controllers, } return o } // AddFlags adds flags related to generic for controller manager to the specified FlagSet. -func (o *GenericComponentConfigOptions) AddFlags(fs *pflag.FlagSet) { +func (o *GenericControllerManagerConfigurationOptions) AddFlags(fss *apiserverflag.NamedFlagSets, allControllers, disabledByDefaultControllers []string) { if o == nil { return } - fs.DurationVar(&o.MinResyncPeriod.Duration, "min-resync-period", o.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.") - fs.StringVar(&o.ContentType, "kube-api-content-type", o.ContentType, "Content type of requests sent to apiserver.") - fs.Float32Var(&o.KubeAPIQPS, "kube-api-qps", o.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.") - fs.Int32Var(&o.KubeAPIBurst, "kube-api-burst", o.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.") - fs.DurationVar(&o.ControllerStartInterval.Duration, "controller-start-interval", o.ControllerStartInterval.Duration, "Interval between starting controller managers.") + o.Debugging.AddFlags(fss.FlagSet("debugging")) + genericfs := fss.FlagSet("generic") + genericfs.DurationVar(&o.MinResyncPeriod.Duration, "min-resync-period", o.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.") + genericfs.StringVar(&o.ClientConnection.ContentType, "kube-api-content-type", o.ClientConnection.ContentType, "Content type of requests sent to apiserver.") + genericfs.Float32Var(&o.ClientConnection.QPS, "kube-api-qps", o.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver.") + genericfs.Int32Var(&o.ClientConnection.Burst, "kube-api-burst", o.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver.") + genericfs.DurationVar(&o.ControllerStartInterval.Duration, "controller-start-interval", o.ControllerStartInterval.Duration, "Interval between starting controller managers.") + // TODO: complete the work of the cloud-controller-manager (and possibly other consumers of this code) respecting the --controllers flag + genericfs.StringSliceVar(&o.Controllers, "controllers", o.Controllers, fmt.Sprintf(""+ + "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ + "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", + strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) - leaderelectionconfig.BindFlags(&o.LeaderElection, fs) + leaderelectionconfig.BindFlags(&o.LeaderElection, genericfs) } // ApplyTo fills up generic config with options. -func (o *GenericComponentConfigOptions) ApplyTo(cfg *componentconfig.GenericComponentConfiguration) error { +func (o *GenericControllerManagerConfigurationOptions) ApplyTo(cfg *componentconfig.GenericControllerManagerConfiguration) error { if o == nil { return nil } + if err := o.Debugging.ApplyTo(&cfg.Debugging); err != nil { + return err + } + + cfg.Port = o.Port + cfg.Address = o.Address cfg.MinResyncPeriod = o.MinResyncPeriod - cfg.ContentType = o.ContentType - cfg.KubeAPIQPS = o.KubeAPIQPS - cfg.KubeAPIBurst = o.KubeAPIBurst + cfg.ClientConnection = o.ClientConnection cfg.ControllerStartInterval = o.ControllerStartInterval cfg.LeaderElection = o.LeaderElection + cfg.Controllers = o.Controllers return nil } // Validate checks validation of GenericOptions. -func (o *GenericComponentConfigOptions) Validate() []error { +func (o *GenericControllerManagerConfigurationOptions) Validate(allControllers []string, disabledByDefaultControllers []string) []error { if o == nil { return nil } errs := []error{} + errs = append(errs, o.Debugging.Validate()...) + + allControllersSet := sets.NewString(allControllers...) + for _, controller := range o.Controllers { + if controller == "*" { + continue + } + if strings.HasPrefix(controller, "-") { + controller = controller[1:] + } + if !allControllersSet.Has(controller) { + errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", controller)) + } + } + return errs } diff --git a/cmd/controller-manager/app/options/kubecloudshared.go b/cmd/controller-manager/app/options/kubecloudshared.go index 92135bd2700..2d777915f92 100644 --- a/cmd/controller-manager/app/options/kubecloudshared.go +++ b/cmd/controller-manager/app/options/kubecloudshared.go @@ -26,8 +26,8 @@ import ( // KubeCloudSharedOptions holds the options shared between kube-controller-manager // and cloud-controller-manager. type KubeCloudSharedOptions struct { - Port int32 - Address string + CloudProvider *CloudProviderOptions + ExternalCloudVolumePlugin string UseServiceAccountCredentials bool AllowUntaggedCloud bool RouteReconciliationPeriod metav1.Duration @@ -45,12 +45,13 @@ type KubeCloudSharedOptions struct { // be made here. Any individual changes should be made in that controller. func NewKubeCloudSharedOptions(cfg componentconfig.KubeCloudSharedConfiguration) *KubeCloudSharedOptions { o := &KubeCloudSharedOptions{ - Port: cfg.Port, - Address: cfg.Address, - RouteReconciliationPeriod: cfg.RouteReconciliationPeriod, - NodeMonitorPeriod: cfg.NodeMonitorPeriod, - ClusterName: cfg.ClusterName, - ConfigureCloudRoutes: cfg.ConfigureCloudRoutes, + CloudProvider: &CloudProviderOptions{}, + ExternalCloudVolumePlugin: cfg.ExternalCloudVolumePlugin, + UseServiceAccountCredentials: cfg.UseServiceAccountCredentials, + RouteReconciliationPeriod: cfg.RouteReconciliationPeriod, + NodeMonitorPeriod: cfg.NodeMonitorPeriod, + ClusterName: cfg.ClusterName, + ConfigureCloudRoutes: cfg.ConfigureCloudRoutes, } return o @@ -62,6 +63,8 @@ func (o *KubeCloudSharedOptions) AddFlags(fs *pflag.FlagSet) { return } + o.CloudProvider.AddFlags(fs) + fs.StringVar(&o.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", o.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.") fs.BoolVar(&o.UseServiceAccountCredentials, "use-service-account-credentials", o.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") fs.BoolVar(&o.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.") @@ -86,8 +89,10 @@ func (o *KubeCloudSharedOptions) ApplyTo(cfg *componentconfig.KubeCloudSharedCon return nil } - cfg.Port = o.Port - cfg.Address = o.Address + if err := o.CloudProvider.ApplyTo(&cfg.CloudProvider); err != nil { + return err + } + cfg.ExternalCloudVolumePlugin = o.ExternalCloudVolumePlugin cfg.UseServiceAccountCredentials = o.UseServiceAccountCredentials cfg.AllowUntaggedCloud = o.AllowUntaggedCloud cfg.RouteReconciliationPeriod = o.RouteReconciliationPeriod @@ -109,5 +114,7 @@ func (o *KubeCloudSharedOptions) Validate() []error { } errs := []error{} + errs = append(errs, o.CloudProvider.Validate()...) + return errs } diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index f3e87e123db..ba1df52136f 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -133,7 +133,7 @@ controller, and serviceaccounts controller.`, func ResyncPeriod(c *config.CompletedConfig) func() time.Duration { return func() time.Duration { factor := rand.Float64() + 1 - return time.Duration(float64(c.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) + return time.Duration(float64(c.ComponentConfig.Generic.MinResyncPeriod.Nanoseconds()) * factor) } } @@ -152,14 +152,14 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { // unsecuredMux is the handler for these controller *after* authn/authz filters have been applied var unsecuredMux *mux.PathRecorderMux if c.SecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } if c.InsecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { @@ -203,7 +203,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { select {} } - if !c.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { + if !c.ComponentConfig.Generic.LeaderElection.LeaderElect { run(context.TODO()) panic("unreachable") } @@ -215,7 +215,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { // add a uniquifier so that two processes on the same host don't accidentally both become active id = id + "_" + string(uuid.NewUUID()) - rl, err := resourcelock.New(c.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, + rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock, "kube-system", "kube-controller-manager", c.LeaderElectionClient.CoreV1(), @@ -229,9 +229,9 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ Lock: rl, - LeaseDuration: c.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, - RenewDeadline: c.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, - RetryPeriod: c.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, + LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration, + RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration, + RetryPeriod: c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { @@ -283,7 +283,7 @@ type ControllerContext struct { } func (c ControllerContext) IsControllerEnabled(name string) bool { - return IsControllerEnabled(name, ControllersDisabledByDefault, c.ComponentConfig.Controllers...) + return IsControllerEnabled(name, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers...) } func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, controllers ...string) bool { @@ -436,8 +436,8 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien return ControllerContext{}, err } - cloud, loopMode, err := createCloudProvider(s.ComponentConfig.CloudProvider.Name, s.ComponentConfig.ExternalCloudVolumePlugin, - s.ComponentConfig.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers) + cloud, loopMode, err := createCloudProvider(s.ComponentConfig.KubeCloudShared.CloudProvider.Name, s.ComponentConfig.KubeCloudShared.ExternalCloudVolumePlugin, + s.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers) if err != nil { return ControllerContext{}, err } @@ -476,7 +476,7 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co continue } - time.Sleep(wait.Jitter(ctx.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(ctx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) glog.V(1).Infof("Starting %q", controllerName) debugHandler, started, err := initFn(ctx) diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index ce8157de074..92700988766 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -96,10 +96,10 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) } } - if len(strings.TrimSpace(ctx.ComponentConfig.NodeIpamController.ServiceCIDR)) != 0 { - _, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.NodeIpamController.ServiceCIDR) + if len(strings.TrimSpace(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR)) != 0 { + _, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIpamController.ServiceCIDR, err) + glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIPAMController.ServiceCIDR, err) } } @@ -109,7 +109,7 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) ctx.ClientBuilder.ClientOrDie("node-controller"), clusterCIDR, serviceCIDR, - int(ctx.ComponentConfig.NodeIpamController.NodeCIDRMaskSize), + int(ctx.ComponentConfig.NodeIPAMController.NodeCIDRMaskSize), ipam.CIDRAllocatorType(ctx.ComponentConfig.KubeCloudShared.CIDRAllocatorType), ) if err != nil { @@ -247,7 +247,7 @@ func startEndpointController(ctx ControllerContext) (http.Handler, bool, error) ctx.InformerFactory.Core().V1().Services(), ctx.InformerFactory.Core().V1().Endpoints(), ctx.ClientBuilder.ClientOrDie("endpoint-controller"), - ).Run(int(ctx.ComponentConfig.EndPointController.ConcurrentEndpointSyncs), ctx.Stop) + ).Run(int(ctx.ComponentConfig.EndpointController.ConcurrentEndpointSyncs), ctx.Stop) return nil, true, nil } diff --git a/cmd/kube-controller-manager/app/options/endpointcontroller.go b/cmd/kube-controller-manager/app/options/endpointcontroller.go index 37f7f79405d..7db7b0e3116 100644 --- a/cmd/kube-controller-manager/app/options/endpointcontroller.go +++ b/cmd/kube-controller-manager/app/options/endpointcontroller.go @@ -22,13 +22,13 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// EndPointControllerOptions holds the EndPointController options. -type EndPointControllerOptions struct { +// EndpointControllerOptions holds the EndPointController options. +type EndpointControllerOptions struct { ConcurrentEndpointSyncs int32 } // AddFlags adds flags related to EndPointController for controller manager to the specified FlagSet. -func (o *EndPointControllerOptions) AddFlags(fs *pflag.FlagSet) { +func (o *EndpointControllerOptions) AddFlags(fs *pflag.FlagSet) { if o == nil { return } @@ -37,7 +37,7 @@ func (o *EndPointControllerOptions) AddFlags(fs *pflag.FlagSet) { } // ApplyTo fills up EndPointController config with options. -func (o *EndPointControllerOptions) ApplyTo(cfg *componentconfig.EndPointControllerConfiguration) error { +func (o *EndpointControllerOptions) ApplyTo(cfg *componentconfig.EndpointControllerConfiguration) error { if o == nil { return nil } @@ -47,8 +47,8 @@ func (o *EndPointControllerOptions) ApplyTo(cfg *componentconfig.EndPointControl return nil } -// Validate checks validation of EndPointControllerOptions. -func (o *EndPointControllerOptions) Validate() []error { +// Validate checks validation of EndpointControllerOptions. +func (o *EndpointControllerOptions) Validate() []error { if o == nil { return nil } diff --git a/cmd/kube-controller-manager/app/options/nodeipamcontroller.go b/cmd/kube-controller-manager/app/options/nodeipamcontroller.go index 3bfad216eb6..0f36188a279 100644 --- a/cmd/kube-controller-manager/app/options/nodeipamcontroller.go +++ b/cmd/kube-controller-manager/app/options/nodeipamcontroller.go @@ -22,14 +22,14 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// NodeIpamControllerOptions holds the NodeIpamController options. -type NodeIpamControllerOptions struct { +// NodeIPAMControllerOptions holds the NodeIpamController options. +type NodeIPAMControllerOptions struct { ServiceCIDR string NodeCIDRMaskSize int32 } // AddFlags adds flags related to NodeIpamController for controller manager to the specified FlagSet. -func (o *NodeIpamControllerOptions) AddFlags(fs *pflag.FlagSet) { +func (o *NodeIPAMControllerOptions) AddFlags(fs *pflag.FlagSet) { if o == nil { return } @@ -39,7 +39,7 @@ func (o *NodeIpamControllerOptions) AddFlags(fs *pflag.FlagSet) { } // ApplyTo fills up NodeIpamController config with options. -func (o *NodeIpamControllerOptions) ApplyTo(cfg *componentconfig.NodeIpamControllerConfiguration) error { +func (o *NodeIPAMControllerOptions) ApplyTo(cfg *componentconfig.NodeIPAMControllerConfiguration) error { if o == nil { return nil } @@ -50,8 +50,8 @@ func (o *NodeIpamControllerOptions) ApplyTo(cfg *componentconfig.NodeIpamControl return nil } -// Validate checks validation of NodeIpamControllerOptions. -func (o *NodeIpamControllerOptions) Validate() []error { +// Validate checks validation of NodeIPAMControllerOptions. +func (o *NodeIPAMControllerOptions) Validate() []error { if o == nil { return nil } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 862e0d9e826..f7c9c4a260c 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -21,12 +21,10 @@ package options import ( "fmt" "net" - "strings" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" apiserverflag "k8s.io/apiserver/pkg/util/flag" @@ -56,9 +54,7 @@ const ( // KubeControllerManagerOptions is the main context object for the kube-controller manager. type KubeControllerManagerOptions struct { - CloudProvider *cmoptions.CloudProviderOptions - Debugging *cmoptions.DebuggingOptions - GenericComponent *cmoptions.GenericComponentConfigOptions + Generic *cmoptions.GenericControllerManagerConfigurationOptions KubeCloudShared *cmoptions.KubeCloudSharedOptions ServiceController *cmoptions.ServiceControllerOptions @@ -67,12 +63,12 @@ type KubeControllerManagerOptions struct { DaemonSetController *DaemonSetControllerOptions DeploymentController *DeploymentControllerOptions DeprecatedFlags *DeprecatedControllerOptions - EndPointController *EndPointControllerOptions + EndpointController *EndpointControllerOptions GarbageCollectorController *GarbageCollectorControllerOptions HPAController *HPAControllerOptions JobController *JobControllerOptions NamespaceController *NamespaceControllerOptions - NodeIpamController *NodeIpamControllerOptions + NodeIPAMController *NodeIPAMControllerOptions NodeLifecycleController *NodeLifecycleControllerOptions PersistentVolumeBinderController *PersistentVolumeBinderControllerOptions PodGCController *PodGCControllerOptions @@ -81,9 +77,6 @@ type KubeControllerManagerOptions struct { ResourceQuotaController *ResourceQuotaControllerOptions SAController *SAControllerOptions - Controllers []string - ExternalCloudVolumePlugin string - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback // TODO: remove insecure serving mode InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback @@ -102,10 +95,8 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { } s := KubeControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{}, - Debugging: &cmoptions.DebuggingOptions{}, - GenericComponent: cmoptions.NewGenericComponentConfigOptions(componentConfig.GenericComponent), - KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), + Generic: cmoptions.NewGenericControllerManagerConfigurationOptions(componentConfig.Generic), + KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), AttachDetachController: &AttachDetachControllerOptions{ ReconcilerSyncLoopPeriod: componentConfig.AttachDetachController.ReconcilerSyncLoopPeriod, }, @@ -124,8 +115,8 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { DeprecatedFlags: &DeprecatedControllerOptions{ RegisterRetryCount: componentConfig.DeprecatedController.RegisterRetryCount, }, - EndPointController: &EndPointControllerOptions{ - ConcurrentEndpointSyncs: componentConfig.EndPointController.ConcurrentEndpointSyncs, + EndpointController: &EndpointControllerOptions{ + ConcurrentEndpointSyncs: componentConfig.EndpointController.ConcurrentEndpointSyncs, }, GarbageCollectorController: &GarbageCollectorControllerOptions{ ConcurrentGCSyncs: componentConfig.GarbageCollectorController.ConcurrentGCSyncs, @@ -147,8 +138,8 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { NamespaceSyncPeriod: componentConfig.NamespaceController.NamespaceSyncPeriod, ConcurrentNamespaceSyncs: componentConfig.NamespaceController.ConcurrentNamespaceSyncs, }, - NodeIpamController: &NodeIpamControllerOptions{ - NodeCIDRMaskSize: componentConfig.NodeIpamController.NodeCIDRMaskSize, + NodeIPAMController: &NodeIPAMControllerOptions{ + NodeCIDRMaskSize: componentConfig.NodeIPAMController.NodeCIDRMaskSize, }, NodeLifecycleController: &NodeLifecycleControllerOptions{ EnableTaintManager: componentConfig.NodeLifecycleController.EnableTaintManager, @@ -179,11 +170,10 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, }, - Controllers: componentConfig.Controllers, SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), - BindPort: int(componentConfig.KubeCloudShared.Port), + BindAddress: net.ParseIP(componentConfig.Generic.Address), + BindPort: int(componentConfig.Generic.Port), BindNetwork: "tcp", }).WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), @@ -225,15 +215,14 @@ func NewDefaultComponentConfig(insecurePort int32) (componentconfig.KubeControll if err := scheme.Convert(&versioned, &internal, nil); err != nil { return internal, err } - internal.KubeCloudShared.Port = insecurePort + internal.Generic.Port = insecurePort return internal, nil } // Flags returns flags for a specific APIServer by section name -func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledByDefaultControllers []string) (fss apiserverflag.NamedFlagSets) { - s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) - s.Debugging.AddFlags(fss.FlagSet("debugging")) - s.GenericComponent.AddFlags(fss.FlagSet("generic")) +func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledByDefaultControllers []string) apiserverflag.NamedFlagSets { + fss := apiserverflag.NamedFlagSets{} + s.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers) s.KubeCloudShared.AddFlags(fss.FlagSet("generic")) s.ServiceController.AddFlags(fss.FlagSet("service controller")) @@ -247,12 +236,12 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy s.DeploymentController.AddFlags(fss.FlagSet("deployment controller")) s.DaemonSetController.AddFlags(fss.FlagSet("daemonset controller")) s.DeprecatedFlags.AddFlags(fss.FlagSet("deprecated")) - s.EndPointController.AddFlags(fss.FlagSet("endpoint controller")) + s.EndpointController.AddFlags(fss.FlagSet("endpoint controller")) s.GarbageCollectorController.AddFlags(fss.FlagSet("garbagecollector controller")) s.HPAController.AddFlags(fss.FlagSet("horizontalpodautoscaling controller")) s.JobController.AddFlags(fss.FlagSet("job controller")) s.NamespaceController.AddFlags(fss.FlagSet("namespace controller")) - s.NodeIpamController.AddFlags(fss.FlagSet("nodeipam controller")) + s.NodeIPAMController.AddFlags(fss.FlagSet("nodeipam controller")) s.NodeLifecycleController.AddFlags(fss.FlagSet("nodelifecycle controller")) s.PersistentVolumeBinderController.AddFlags(fss.FlagSet("persistentvolume-binder controller")) s.PodGCController.AddFlags(fss.FlagSet("podgc controller")) @@ -264,11 +253,6 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+ - "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ - "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", - strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) - fs.StringVar(&s.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.") var dummy string fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.") fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.") @@ -279,13 +263,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy // ApplyTo fills up controller manager config with options. func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) error { - if err := s.CloudProvider.ApplyTo(&c.ComponentConfig.CloudProvider); err != nil { - return err - } - if err := s.Debugging.ApplyTo(&c.ComponentConfig.Debugging); err != nil { - return err - } - if err := s.GenericComponent.ApplyTo(&c.ComponentConfig.GenericComponent); err != nil { + if err := s.Generic.ApplyTo(&c.ComponentConfig.Generic); err != nil { return err } if err := s.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { @@ -306,7 +284,7 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e if err := s.DeprecatedFlags.ApplyTo(&c.ComponentConfig.DeprecatedController); err != nil { return err } - if err := s.EndPointController.ApplyTo(&c.ComponentConfig.EndPointController); err != nil { + if err := s.EndpointController.ApplyTo(&c.ComponentConfig.EndpointController); err != nil { return err } if err := s.GarbageCollectorController.ApplyTo(&c.ComponentConfig.GarbageCollectorController); err != nil { @@ -321,7 +299,7 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e if err := s.NamespaceController.ApplyTo(&c.ComponentConfig.NamespaceController); err != nil { return err } - if err := s.NodeIpamController.ApplyTo(&c.ComponentConfig.NodeIpamController); err != nil { + if err := s.NodeIPAMController.ApplyTo(&c.ComponentConfig.NodeIPAMController); err != nil { return err } if err := s.NodeLifecycleController.ApplyTo(&c.ComponentConfig.NodeLifecycleController); err != nil { @@ -365,11 +343,8 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e // sync back to component config // TODO: find more elegant way than syncing back the values. - c.ComponentConfig.KubeCloudShared.Port = int32(s.InsecureServing.BindPort) - c.ComponentConfig.KubeCloudShared.Address = s.InsecureServing.BindAddress.String() - - c.ComponentConfig.Controllers = s.Controllers - c.ComponentConfig.ExternalCloudVolumePlugin = s.ExternalCloudVolumePlugin + c.ComponentConfig.Generic.Port = int32(s.InsecureServing.BindPort) + c.ComponentConfig.Generic.Address = s.InsecureServing.BindAddress.String() return nil } @@ -378,21 +353,19 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e func (s *KubeControllerManagerOptions) Validate(allControllers []string, disabledByDefaultControllers []string) error { var errs []error - errs = append(errs, s.CloudProvider.Validate()...) - errs = append(errs, s.Debugging.Validate()...) - errs = append(errs, s.GenericComponent.Validate()...) + errs = append(errs, s.Generic.Validate(allControllers, disabledByDefaultControllers)...) errs = append(errs, s.KubeCloudShared.Validate()...) errs = append(errs, s.AttachDetachController.Validate()...) errs = append(errs, s.CSRSigningController.Validate()...) errs = append(errs, s.DaemonSetController.Validate()...) errs = append(errs, s.DeploymentController.Validate()...) errs = append(errs, s.DeprecatedFlags.Validate()...) - errs = append(errs, s.EndPointController.Validate()...) + errs = append(errs, s.EndpointController.Validate()...) errs = append(errs, s.GarbageCollectorController.Validate()...) errs = append(errs, s.HPAController.Validate()...) errs = append(errs, s.JobController.Validate()...) errs = append(errs, s.NamespaceController.Validate()...) - errs = append(errs, s.NodeIpamController.Validate()...) + errs = append(errs, s.NodeIPAMController.Validate()...) errs = append(errs, s.NodeLifecycleController.Validate()...) errs = append(errs, s.PersistentVolumeBinderController.Validate()...) errs = append(errs, s.PodGCController.Validate()...) @@ -408,20 +381,6 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable // TODO: validate component config, master and kubeconfig - allControllersSet := sets.NewString(allControllers...) - for _, controller := range s.Controllers { - if controller == "*" { - continue - } - if strings.HasPrefix(controller, "-") { - controller = controller[1:] - } - - if !allControllersSet.Has(controller) { - errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", controller)) - } - } - return utilerrors.NewAggregate(errs) } @@ -439,9 +398,9 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy if err != nil { return nil, err } - kubeconfig.ContentConfig.ContentType = s.GenericComponent.ContentType - kubeconfig.QPS = s.GenericComponent.KubeAPIQPS - kubeconfig.Burst = int(s.GenericComponent.KubeAPIBurst) + kubeconfig.ContentConfig.ContentType = s.Generic.ClientConnection.ContentType + kubeconfig.QPS = s.Generic.ClientConnection.QPS + kubeconfig.Burst = int(s.Generic.ClientConnection.Burst) client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, KubeControllerManagerUserAgent)) if err != nil { @@ -450,7 +409,7 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy // shallow copy, do not modify the kubeconfig.Timeout. config := *kubeconfig - config.Timeout = s.GenericComponent.LeaderElection.RenewDeadline.Duration + config.Timeout = s.Generic.LeaderElection.RenewDeadline.Duration leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "leader-election")) eventRecorder := createRecorder(client, KubeControllerManagerUserAgent) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index e44102e6632..f23bc13b3a8 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/pflag" + apimachineryconfig "k8s.io/apimachinery/pkg/apis/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" apiserverconfig "k8s.io/apiserver/pkg/apis/config" @@ -121,19 +122,15 @@ func TestAddFlags(t *testing.T) { sort.Sort(sortedGCIgnoredResources(s.GarbageCollectorController.GCIgnoredResources)) expected := &KubeControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableProfiling: false, - EnableContentionProfiling: true, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, - ContentType: "application/json", - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, + Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ + Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, + ClientConnection: apimachineryconfig.ClientConnectionConfiguration{ + ContentType: "application/json", + QPS: 50.0, + Burst: 100, + }, ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, LeaderElection: apiserverconfig.LeaderElectionConfiguration{ ResourceLock: "configmap", @@ -142,10 +139,13 @@ func TestAddFlags(t *testing.T) { RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, }, + Debugging: &cmoptions.DebuggingOptions{ + EnableProfiling: false, + EnableContentionProfiling: true, + }, + Controllers: []string{"foo", "bar"}, }, KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10252, // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: DeprecatedInsecureServingOptions.ApplyTo will write the flag value back into the component config UseServiceAccountCredentials: true, RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, @@ -154,6 +154,10 @@ func TestAddFlags(t *testing.T) { AllocateNodeCIDRs: true, CIDRAllocatorType: "CloudAllocator", ConfigureCloudRoutes: false, + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "gce", + CloudConfigFile: "/cloud-config", + }, }, AttachDetachController: &AttachDetachControllerOptions{ ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, @@ -175,7 +179,7 @@ func TestAddFlags(t *testing.T) { DeletingPodsQPS: 0.1, RegisterRetryCount: 10, }, - EndPointController: &EndPointControllerOptions{ + EndpointController: &EndpointControllerOptions{ ConcurrentEndpointSyncs: 10, }, GarbageCollectorController: &GarbageCollectorControllerOptions{ @@ -201,7 +205,7 @@ func TestAddFlags(t *testing.T) { NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, ConcurrentNamespaceSyncs: 20, }, - NodeIpamController: &NodeIpamControllerOptions{ + NodeIPAMController: &NodeIPAMControllerOptions{ NodeCIDRMaskSize: 48, }, NodeLifecycleController: &NodeLifecycleControllerOptions{ @@ -249,7 +253,6 @@ func TestAddFlags(t *testing.T) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: 2, }, - Controllers: []string{"foo", "bar"}, SecureServing: (&apiserveroptions.SecureServingOptions{ BindPort: 10001, BindAddress: net.ParseIP("192.168.4.21"),