diff --git a/cmd/cloud-controller-manager/app/config/BUILD b/cmd/cloud-controller-manager/app/config/BUILD index 5f4ce185b20..157c45435eb 100644 --- a/cmd/cloud-controller-manager/app/config/BUILD +++ b/cmd/cloud-controller-manager/app/config/BUILD @@ -5,7 +5,14 @@ go_library( srcs = ["config.go"], importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config", visibility = ["//visibility:public"], - deps = ["//cmd/controller-manager/app:go_default_library"], + deps = [ + "//cmd/controller-manager/app:go_default_library", + "//pkg/apis/componentconfig:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + ], ) filegroup( diff --git a/cmd/cloud-controller-manager/app/config/config.go b/cmd/cloud-controller-manager/app/config/config.go index ca31d87cd61..3d201e47546 100644 --- a/cmd/cloud-controller-manager/app/config/config.go +++ b/cmd/cloud-controller-manager/app/config/config.go @@ -17,25 +17,39 @@ limitations under the License. package app import ( - "time" - + apiserver "k8s.io/apiserver/pkg/server" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app" + "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// ExtraConfig are part of Config, also can place your custom config here. -type ExtraConfig struct { - NodeStatusUpdateFrequency time.Duration -} - // Config is the main context object for the cloud controller manager. type Config struct { - Generic genericcontrollermanager.Config - Extra ExtraConfig + ComponentConfig componentconfig.CloudControllerManagerConfiguration + + SecureServing *apiserver.SecureServingInfo + // TODO: remove deprecated insecure serving + InsecureServing *genericcontrollermanager.InsecureServingInfo + Authentication apiserver.AuthenticationInfo + Authorization apiserver.AuthorizationInfo + + // the general kube client + Client *clientset.Clientset + + // the client only used for leader election + LeaderElectionClient *clientset.Clientset + + // the rest config for the master + Kubeconfig *restclient.Config + + // the event sink + EventRecorder record.EventRecorder } type completedConfig struct { - Generic genericcontrollermanager.CompletedConfig - Extra *ExtraConfig + *Config } // CompletedConfig same as Config, just to swap private object. @@ -46,10 +60,6 @@ type CompletedConfig struct { // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() *CompletedConfig { - cc := completedConfig{ - c.Generic.Complete(), - &c.Extra, - } - + cc := completedConfig{c} return &CompletedConfig{&cc} } diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 8589ba35d5c..988bf5edf36 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -85,13 +85,13 @@ the cloud specific control loops shipped with Kubernetes.`, func resyncPeriod(c *cloudcontrollerconfig.CompletedConfig) func() time.Duration { return func() time.Duration { factor := rand.Float64() + 1 - return time.Duration(float64(c.Generic.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) + return time.Duration(float64(c.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) } } // Run runs the ExternalCMServer. This should never exit. func Run(c *cloudcontrollerconfig.CompletedConfig) error { - cloud, err := cloudprovider.InitCloudProvider(c.Generic.ComponentConfig.CloudProvider.Name, c.Generic.ComponentConfig.CloudProvider.CloudConfigFile) + cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.CloudProvider.Name, c.ComponentConfig.CloudProvider.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } @@ -100,7 +100,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { } if cloud.HasClusterID() == false { - if c.Generic.ComponentConfig.KubeCloudShared.AllowUntaggedCloud == true { + if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud == true { glog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") } else { glog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") @@ -109,38 +109,38 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { // setup /configz endpoint if cz, err := configz.New("componentconfig"); err == nil { - cz.Set(c.Generic.ComponentConfig) + cz.Set(c.ComponentConfig) } else { glog.Errorf("unable to register configz: %c", err) } // Start the controller manager HTTP server stopCh := make(chan struct{}) - if c.Generic.SecureServing != nil { - handler := genericcontrollermanager.NewBaseHandler(&c.Generic) - handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Generic) - if err := c.Generic.SecureServing.Serve(handler, 0, stopCh); err != nil { + if c.SecureServing != nil { + handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication) + if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } - if c.Generic.InsecureServing != nil { - handler := genericcontrollermanager.NewBaseHandler(&c.Generic) - handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Generic) - if err := c.Generic.InsecureServing.Serve(handler, 0, stopCh); err != nil { + if c.InsecureServing != nil { + handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication) + if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { return err } } run := func(stop <-chan struct{}) { rootClientBuilder := controller.SimpleControllerClientBuilder{ - ClientConfig: c.Generic.Kubeconfig, + ClientConfig: c.Kubeconfig, } var clientBuilder controller.ControllerClientBuilder - if c.Generic.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials { + if c.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials { clientBuilder = controller.SAControllerClientBuilder{ - ClientConfig: restclient.AnonymousClientConfig(c.Generic.Kubeconfig), - CoreClient: c.Generic.Client.CoreV1(), - AuthenticationClient: c.Generic.Client.AuthenticationV1(), + ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), + CoreClient: c.Client.CoreV1(), + AuthenticationClient: c.Client.AuthenticationV1(), Namespace: "kube-system", } } else { @@ -152,7 +152,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { } } - if !c.Generic.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { + if !c.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { run(nil) panic("unreachable") } @@ -166,13 +166,13 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { id = id + "_" + string(uuid.NewUUID()) // Lock required for leader election - rl, err := resourcelock.New(c.Generic.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, + rl, err := resourcelock.New(c.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, "kube-system", "cloud-controller-manager", - c.Generic.LeaderElectionClient.CoreV1(), + c.LeaderElectionClient.CoreV1(), resourcelock.ResourceLockConfig{ Identity: id, - EventRecorder: c.Generic.EventRecorder, + EventRecorder: c.EventRecorder, }) if err != nil { glog.Fatalf("error creating lock: %v", err) @@ -181,9 +181,9 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { // Try and become the leader and start cloud controller manager loops leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ Lock: rl, - LeaseDuration: c.Generic.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, - RenewDeadline: c.Generic.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, - RetryPeriod: c.Generic.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, + LeaseDuration: c.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, + RenewDeadline: c.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, + RetryPeriod: c.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { @@ -213,16 +213,16 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, rootClientBuilde nodeController := cloudcontrollers.NewCloudNodeController( sharedInformers.Core().V1().Nodes(), client("cloud-node-controller"), cloud, - c.Generic.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - c.Extra.NodeStatusUpdateFrequency) + c.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, + c.ComponentConfig.NodeStatusUpdateFrequency.Duration) nodeController.Run(stop) - time.Sleep(wait.Jitter(c.Generic.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) // Start the PersistentVolumeLabelController pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud) go pvlController.Run(5, stop) - time.Sleep(wait.Jitter(c.Generic.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) // Start the service controller serviceController, err := servicecontroller.New( @@ -230,34 +230,34 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, rootClientBuilde client("service-controller"), sharedInformers.Core().V1().Services(), sharedInformers.Core().V1().Nodes(), - c.Generic.ComponentConfig.KubeCloudShared.ClusterName, + c.ComponentConfig.KubeCloudShared.ClusterName, ) if err != nil { glog.Errorf("Failed to start service controller: %v", err) } else { - go serviceController.Run(stop, int(c.Generic.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) - time.Sleep(wait.Jitter(c.Generic.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + go serviceController.Run(stop, int(c.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) + time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) } // If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller - if c.Generic.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs && c.Generic.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { + if c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs && c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { if routes, ok := cloud.Routes(); !ok { glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") } else { var clusterCIDR *net.IPNet - if len(strings.TrimSpace(c.Generic.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 { - _, clusterCIDR, err = net.ParseCIDR(c.Generic.ComponentConfig.KubeCloudShared.ClusterCIDR) + if len(strings.TrimSpace(c.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 { + _, clusterCIDR, err = net.ParseCIDR(c.ComponentConfig.KubeCloudShared.ClusterCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.Generic.ComponentConfig.KubeCloudShared.ClusterCIDR, err) + glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.ComponentConfig.KubeCloudShared.ClusterCIDR, err) } } - routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), c.Generic.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDR) - go routeController.Run(stop, c.Generic.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) - time.Sleep(wait.Jitter(c.Generic.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) + routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), c.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDR) + go routeController.Run(stop, c.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) + time.Sleep(wait.Jitter(c.ComponentConfig.GenericComponent.ControllerStartInterval.Duration, ControllerStartJitter)) } } else { - glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.Generic.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, c.Generic.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) + glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) } // If apiserver is not running we should wait for some time and fail only then. This is particularly diff --git a/cmd/cloud-controller-manager/app/options/BUILD b/cmd/cloud-controller-manager/app/options/BUILD index 71b86717784..7e36ffa8d08 100644 --- a/cmd/cloud-controller-manager/app/options/BUILD +++ b/cmd/cloud-controller-manager/app/options/BUILD @@ -13,12 +13,24 @@ go_library( deps = [ "//cmd/cloud-controller-manager/app/config:go_default_library", "//cmd/controller-manager/app/options:go_default_library", + "//pkg/api/legacyscheme:go_default_library", + "//pkg/apis/componentconfig:go_default_library", + "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/features:go_default_library", "//pkg/master/ports:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index cfde5f3b7e6..433d9935164 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -18,24 +18,49 @@ package options import ( "fmt" - "time" + "net" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" + apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config" cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" + "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/apis/componentconfig" + componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/master/ports" - // add the kubernetes feature gates _ "k8s.io/kubernetes/pkg/features" + "github.com/golang/glog" "github.com/spf13/pflag" ) // CloudControllerManagerOptions is the main context object for the controller manager. type CloudControllerManagerOptions struct { - Generic *cmoptions.GenericControllerManagerOptions + CloudProvider *cmoptions.CloudProviderOptions + Debugging *cmoptions.DebuggingOptions + GenericComponent *cmoptions.GenericComponentConfigOptions + KubeCloudShared *cmoptions.KubeCloudSharedOptions + ServiceController *cmoptions.ServiceControllerOptions + + SecureServing *apiserveroptions.SecureServingOptions + // TODO: remove insecure serving mode + InsecureServing *cmoptions.InsecureServingOptions + Authentication *apiserveroptions.DelegatingAuthenticationOptions + Authorization *apiserveroptions.DelegatingAuthorizationOptions + + Master string + Kubeconfig string // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status NodeStatusUpdateFrequency metav1.Duration @@ -43,37 +68,125 @@ type CloudControllerManagerOptions struct { // NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config. func NewCloudControllerManagerOptions() *CloudControllerManagerOptions { - componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureCloudControllerManagerPort) + componentConfig := NewDefaultComponentConfig(ports.InsecureCloudControllerManagerPort) s := CloudControllerManagerOptions{ - // The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'. - // Please make common changes there and put anything cloud specific here. - Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig), - NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute}, + CloudProvider: &cmoptions.CloudProviderOptions{}, + Debugging: &cmoptions.DebuggingOptions{}, + GenericComponent: cmoptions.NewGenericComponentConfigOptions(componentConfig.GenericComponent), + KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), + ServiceController: &cmoptions.ServiceControllerOptions{ + ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, + }, + SecureServing: apiserveroptions.NewSecureServingOptions(), + InsecureServing: &cmoptions.InsecureServingOptions{ + BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), + BindPort: int(componentConfig.KubeCloudShared.Port), + BindNetwork: "tcp", + }, + Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() + Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() + NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency, } - s.Generic.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" - s.Generic.SecureServing.ServerCert.PairName = "cloud-controller-manager" + s.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" + s.SecureServing.ServerCert.PairName = "cloud-controller-manager" + + // disable secure serving for now + // TODO: enable HTTPS by default + s.SecureServing.BindPort = 0 return &s } +// NewDefaultComponentConfig returns cloud-controller manager configuration object. +func NewDefaultComponentConfig(insecurePort int32) componentconfig.CloudControllerManagerConfiguration { + scheme := runtime.NewScheme() + componentconfigv1alpha1.AddToScheme(scheme) + componentconfig.AddToScheme(scheme) + + versioned := componentconfigv1alpha1.CloudControllerManagerConfiguration{} + scheme.Default(&versioned) + + internal := componentconfig.CloudControllerManagerConfiguration{} + scheme.Convert(&versioned, &internal, nil) + internal.KubeCloudShared.Port = insecurePort + return internal +} + // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet func (o *CloudControllerManagerOptions) AddFlags(fs *pflag.FlagSet) { - o.Generic.AddFlags(fs) + o.CloudProvider.AddFlags(fs) + o.Debugging.AddFlags(fs) + o.GenericComponent.AddFlags(fs) + o.KubeCloudShared.AddFlags(fs) + o.ServiceController.AddFlags(fs) + o.SecureServing.AddFlags(fs) + o.InsecureServing.AddFlags(fs) + o.Authentication.AddFlags(fs) + o.Authorization.AddFlags(fs) + + fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") + fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") utilfeature.DefaultFeatureGate.AddFlag(fs) } // ApplyTo fills up cloud controller manager config with options. -func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config) error { - if err := o.Generic.ApplyTo(&c.Generic, "cloud-controller-manager"); err != nil { +func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, userAgent string) error { + if err := o.CloudProvider.ApplyTo(&c.ComponentConfig.CloudProvider); err != nil { + return err + } + if err := o.Debugging.ApplyTo(&c.ComponentConfig.Debugging); err != nil { + return err + } + if err := o.GenericComponent.ApplyTo(&c.ComponentConfig.GenericComponent); err != nil { + return err + } + if err := o.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { + return err + } + if err := o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { + return err + } + if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil { + return err + } + if err := o.InsecureServing.ApplyTo(&c.InsecureServing); err != nil { + return err + } + if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { + return err + } + if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { return err } - c.Extra.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency.Duration + // sync back to component config + // TODO: find more elegant way than synching back the values. + c.ComponentConfig.KubeCloudShared.Port = int32(o.InsecureServing.BindPort) + c.ComponentConfig.KubeCloudShared.Address = o.InsecureServing.BindAddress.String() + + var err error + c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig) + if err != nil { + return err + } + c.Kubeconfig.ContentConfig.ContentType = o.GenericComponent.ContentType + c.Kubeconfig.QPS = o.GenericComponent.KubeAPIQPS + c.Kubeconfig.Burst = int(o.GenericComponent.KubeAPIBurst) + + c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent)) + if err != nil { + return err + } + + c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election")) + + c.EventRecorder = createRecorder(c.Client, userAgent) + c.ComponentConfig.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency return nil } @@ -81,9 +194,18 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config) // Validate is used to validate config before launching the cloud controller manager func (o *CloudControllerManagerOptions) Validate() error { errors := []error{} - errors = append(errors, o.Generic.Validate()...) - if len(o.Generic.CloudProvider.Name) == 0 { + errors = append(errors, o.CloudProvider.Validate()...) + errors = append(errors, o.Debugging.Validate()...) + errors = append(errors, o.GenericComponent.Validate()...) + errors = append(errors, o.KubeCloudShared.Validate()...) + errors = append(errors, o.ServiceController.Validate()...) + errors = append(errors, o.SecureServing.Validate()...) + errors = append(errors, o.InsecureServing.Validate()...) + errors = append(errors, o.Authentication.Validate()...) + errors = append(errors, o.Authorization.Validate()...) + + if len(o.CloudProvider.Name) == 0 { errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty")) } @@ -97,9 +219,16 @@ func (o CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, } c := &cloudcontrollerconfig.Config{} - if err := o.ApplyTo(c); err != nil { + if err := o.ApplyTo(c, "cloud-controller-manager"); err != nil { return nil, err } return c, nil } + +func createRecorder(kubeClient kubernetes.Interface, userAgent string) record.EventRecorder { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent}) +} diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index 71c14092e9e..fe2f62f0b35 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -35,139 +35,57 @@ func TestDefaultFlags(t *testing.T) { s := NewCloudControllerManagerOptions() expected := &CloudControllerManagerOptions{ - Generic: &cmoptions.GenericControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "", - CloudConfigFile: "", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableContentionProfiling: false, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 20.0, - KubeAPIBurst: 30, - ControllerStartInterval: metav1.Duration{Duration: 0}, - LeaderElection: componentconfig.LeaderElectionConfiguration{ - ResourceLock: "endpoints", - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - }, - }, - KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "kubernetes", - ClusterCIDR: "", - AllocateNodeCIDRs: false, - CIDRAllocatorType: "", - ConfigureCloudRoutes: true, - }, - AttachDetachController: &cmoptions.AttachDetachControllerOptions{ - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 1 * time.Minute}, - }, - CSRSigningController: &cmoptions.CSRSigningControllerOptions{ - ClusterSigningCertFile: "/etc/kubernetes/ca/ca.pem", - ClusterSigningKeyFile: "/etc/kubernetes/ca/ca.key", - ClusterSigningDuration: metav1.Duration{Duration: 8760 * time.Hour}, - }, - DaemonSetController: &cmoptions.DaemonSetControllerOptions{ - ConcurrentDaemonSetSyncs: 2, - }, - DeploymentController: &cmoptions.DeploymentControllerOptions{ - ConcurrentDeploymentSyncs: 5, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - }, - DeprecatedFlags: &cmoptions.DeprecatedControllerOptions{ - RegisterRetryCount: 10, - }, - EndPointController: &cmoptions.EndPointControllerOptions{ - ConcurrentEndpointSyncs: 5, - }, - GarbageCollectorController: &cmoptions.GarbageCollectorControllerOptions{ - EnableGarbageCollector: true, - ConcurrentGCSyncs: 20, - }, - HPAController: &cmoptions.HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, - }, - JobController: &cmoptions.JobControllerOptions{ - ConcurrentJobSyncs: 5, - }, - NamespaceController: &cmoptions.NamespaceControllerOptions{ - ConcurrentNamespaceSyncs: 10, - NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - }, - NodeIpamController: &cmoptions.NodeIpamControllerOptions{ - NodeCIDRMaskSize: 24, - }, - NodeLifecycleController: &cmoptions.NodeLifecycleControllerOptions{ - EnableTaintManager: true, - NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 1 * time.Minute}, - PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, - }, - PersistentVolumeBinderController: &cmoptions.PersistentVolumeBinderControllerOptions{ - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, - VolumeConfiguration: componentconfig.VolumeConfiguration{ - EnableDynamicProvisioning: true, - EnableHostPathProvisioning: false, - FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", - PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 300, - IncrementTimeoutNFS: 30, - MinimumTimeoutHostPath: 60, - IncrementTimeoutHostPath: 30, - }, - }, - }, - PodGCController: &cmoptions.PodGCControllerOptions{ - TerminatedPodGCThreshold: 12500, - }, - ReplicaSetController: &cmoptions.ReplicaSetControllerOptions{ - ConcurrentRSSyncs: 5, - }, - ReplicationController: &cmoptions.ReplicationControllerOptions{ - ConcurrentRCSyncs: 5, - }, - ResourceQuotaController: &cmoptions.ResourceQuotaControllerOptions{ - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - ConcurrentResourceQuotaSyncs: 5, - }, - SAController: &cmoptions.SAControllerOptions{ - ConcurrentSATokenSyncs: 5, - }, - ServiceController: &cmoptions.ServiceControllerOptions{ - ConcurrentServiceSyncs: 1, - }, - Controllers: []string{"*"}, - SecureServing: &apiserveroptions.SecureServingOptions{ - BindPort: 0, - BindAddress: net.ParseIP("0.0.0.0"), - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/var/run/kubernetes", - PairName: "cloud-controller-manager", - }, - HTTP2MaxStreamsPerConnection: 0, - }, - InsecureServing: &cmoptions.InsecureServingOptions{ - BindAddress: net.ParseIP("0.0.0.0"), - BindPort: int(10253), - BindNetwork: "tcp", - }, - Kubeconfig: "", - Master: "", + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "", + CloudConfigFile: "", }, + Debugging: &cmoptions.DebuggingOptions{ + EnableContentionProfiling: false, + }, + GenericComponent: &cmoptions.GenericComponentConfigOptions{ + MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, + ContentType: "application/vnd.kubernetes.protobuf", + KubeAPIQPS: 20.0, + KubeAPIBurst: 30, + ControllerStartInterval: metav1.Duration{Duration: 0}, + LeaderElection: componentconfig.LeaderElectionConfiguration{ + ResourceLock: "endpoints", + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + }, + KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ + Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ClusterName: "kubernetes", + ClusterCIDR: "", + AllocateNodeCIDRs: false, + CIDRAllocatorType: "", + ConfigureCloudRoutes: true, + }, + ServiceController: &cmoptions.ServiceControllerOptions{ + ConcurrentServiceSyncs: 1, + }, + SecureServing: &apiserveroptions.SecureServingOptions{ + BindPort: 0, + BindAddress: net.ParseIP("0.0.0.0"), + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/var/run/kubernetes", + PairName: "cloud-controller-manager", + }, + HTTP2MaxStreamsPerConnection: 0, + }, + InsecureServing: &cmoptions.InsecureServingOptions{ + BindAddress: net.ParseIP("0.0.0.0"), + BindPort: int(10253), + BindNetwork: "tcp", + }, + Kubeconfig: "", + Master: "", NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute}, } if !reflect.DeepEqual(expected, s) { @@ -216,139 +134,57 @@ func TestAddFlags(t *testing.T) { f.Parse(args) expected := &CloudControllerManagerOptions{ - Generic: &cmoptions.GenericControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableContentionProfiling: true, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - LeaderElection: componentconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, - }, - }, - KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "k8s", - ClusterCIDR: "1.2.3.4/24", - AllocateNodeCIDRs: true, - CIDRAllocatorType: "RangeAllocator", - ConfigureCloudRoutes: false, - }, - AttachDetachController: &cmoptions.AttachDetachControllerOptions{ - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 1 * time.Minute}, - }, - CSRSigningController: &cmoptions.CSRSigningControllerOptions{ - ClusterSigningCertFile: "/etc/kubernetes/ca/ca.pem", - ClusterSigningKeyFile: "/etc/kubernetes/ca/ca.key", - ClusterSigningDuration: metav1.Duration{Duration: 8760 * time.Hour}, - }, - DaemonSetController: &cmoptions.DaemonSetControllerOptions{ - ConcurrentDaemonSetSyncs: 2, - }, - DeploymentController: &cmoptions.DeploymentControllerOptions{ - ConcurrentDeploymentSyncs: 5, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - }, - DeprecatedFlags: &cmoptions.DeprecatedControllerOptions{ - RegisterRetryCount: 10, - }, - EndPointController: &cmoptions.EndPointControllerOptions{ - ConcurrentEndpointSyncs: 5, - }, - GarbageCollectorController: &cmoptions.GarbageCollectorControllerOptions{ - ConcurrentGCSyncs: 20, - EnableGarbageCollector: true, - }, - HPAController: &cmoptions.HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, - }, - JobController: &cmoptions.JobControllerOptions{ - ConcurrentJobSyncs: 5, - }, - NamespaceController: &cmoptions.NamespaceControllerOptions{ - NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - ConcurrentNamespaceSyncs: 10, - }, - NodeIpamController: &cmoptions.NodeIpamControllerOptions{ - NodeCIDRMaskSize: 24, - }, - NodeLifecycleController: &cmoptions.NodeLifecycleControllerOptions{ - EnableTaintManager: true, - NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 1 * time.Minute}, - PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, - }, - PersistentVolumeBinderController: &cmoptions.PersistentVolumeBinderControllerOptions{ - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, - VolumeConfiguration: componentconfig.VolumeConfiguration{ - EnableDynamicProvisioning: true, - EnableHostPathProvisioning: false, - FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", - PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 300, - IncrementTimeoutNFS: 30, - MinimumTimeoutHostPath: 60, - IncrementTimeoutHostPath: 30, - }, - }, - }, - PodGCController: &cmoptions.PodGCControllerOptions{ - TerminatedPodGCThreshold: 12500, - }, - ReplicaSetController: &cmoptions.ReplicaSetControllerOptions{ - ConcurrentRSSyncs: 5, - }, - ReplicationController: &cmoptions.ReplicationControllerOptions{ - ConcurrentRCSyncs: 5, - }, - ResourceQuotaController: &cmoptions.ResourceQuotaControllerOptions{ - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - ConcurrentResourceQuotaSyncs: 5, - }, - SAController: &cmoptions.SAControllerOptions{ - ConcurrentSATokenSyncs: 5, - }, - ServiceController: &cmoptions.ServiceControllerOptions{ - ConcurrentServiceSyncs: 1, - }, - Controllers: []string{"*"}, - SecureServing: &apiserveroptions.SecureServingOptions{ - BindPort: 10001, - BindAddress: net.ParseIP("192.168.4.21"), - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "cloud-controller-manager", - }, - HTTP2MaxStreamsPerConnection: 47, - }, - InsecureServing: &cmoptions.InsecureServingOptions{ - BindAddress: net.ParseIP("192.168.4.10"), - BindPort: int(10000), - BindNetwork: "tcp", - }, - Kubeconfig: "/kubeconfig", - Master: "192.168.4.20", + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "gce", + CloudConfigFile: "/cloud-config", }, + Debugging: &cmoptions.DebuggingOptions{ + EnableContentionProfiling: true, + }, + GenericComponent: &cmoptions.GenericComponentConfigOptions{ + MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, + ContentType: "application/vnd.kubernetes.protobuf", + KubeAPIQPS: 50.0, + KubeAPIBurst: 100, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + LeaderElection: componentconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + }, + KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ + Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ClusterName: "k8s", + ClusterCIDR: "1.2.3.4/24", + AllocateNodeCIDRs: true, + CIDRAllocatorType: "RangeAllocator", + ConfigureCloudRoutes: false, + }, + ServiceController: &cmoptions.ServiceControllerOptions{ + ConcurrentServiceSyncs: 1, + }, + SecureServing: &apiserveroptions.SecureServingOptions{ + BindPort: 10001, + BindAddress: net.ParseIP("192.168.4.21"), + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "cloud-controller-manager", + }, + HTTP2MaxStreamsPerConnection: 47, + }, + InsecureServing: &cmoptions.InsecureServingOptions{ + BindAddress: net.ParseIP("192.168.4.10"), + BindPort: int(10000), + BindNetwork: "tcp", + }, + Kubeconfig: "/kubeconfig", + Master: "192.168.4.20", NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute}, } if !reflect.DeepEqual(expected, s) { diff --git a/cmd/controller-manager/app/BUILD b/cmd/controller-manager/app/BUILD index d36a9526ec5..c0bd8593975 100644 --- a/cmd/controller-manager/app/BUILD +++ b/cmd/controller-manager/app/BUILD @@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ - "config.go", "helper.go", "insecure_serving.go", "serve.go", @@ -25,8 +24,6 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server/mux:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/routes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/cmd/controller-manager/app/config.go b/cmd/controller-manager/app/config.go deleted file mode 100644 index 6a2fc592e12..00000000000 --- a/cmd/controller-manager/app/config.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - apiserver "k8s.io/apiserver/pkg/server" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "k8s.io/kubernetes/pkg/apis/componentconfig" -) - -// Config is the main context object for the controller manager. -type Config struct { - ComponentConfig componentconfig.KubeControllerManagerConfiguration - - SecureServing *apiserver.SecureServingInfo - // TODO: remove deprecated insecure serving - InsecureServing *InsecureServingInfo - Authentication apiserver.AuthenticationInfo - Authorization apiserver.AuthorizationInfo - - // the general kube client - Client *clientset.Clientset - - // the client only used for leader election - LeaderElectionClient *clientset.Clientset - - // the rest config for the master - Kubeconfig *restclient.Config - - // the event sink - EventRecorder record.EventRecorder -} - -type completedConfig struct { - *Config -} - -// CompletedConfig same as Config, just to swap private object. -type CompletedConfig struct { - // Embed a private pointer that cannot be instantiated outside of this package. - *completedConfig -} - -// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() CompletedConfig { - cc := completedConfig{c} - return CompletedConfig{&cc} -} diff --git a/cmd/controller-manager/app/options/BUILD b/cmd/controller-manager/app/options/BUILD index ea9ff4417df..83b848891f1 100644 --- a/cmd/controller-manager/app/options/BUILD +++ b/cmd/controller-manager/app/options/BUILD @@ -20,7 +20,6 @@ go_library( "namespacecontroller.go", "nodeipamcontroller.go", "nodelifecyclecontroller.go", - "options.go", "persistentvolumebindercontroller.go", "podgccontroller.go", "replicasetcontroller.go", @@ -33,21 +32,11 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cmd/controller-manager/app:go_default_library", - "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/componentconfig:go_default_library", - "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/client/leaderelectionconfig:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", - "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/cmd/controller-manager/app/options/csrsigningcontroller.go b/cmd/controller-manager/app/options/csrsigningcontroller.go index f307c503f71..bf55c9df89b 100644 --- a/cmd/controller-manager/app/options/csrsigningcontroller.go +++ b/cmd/controller-manager/app/options/csrsigningcontroller.go @@ -22,6 +22,16 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" ) +const ( + // These defaults are deprecated and exported so that we can warn if + // they are being used. + + // DefaultClusterSigningCertFile is deprecated. Do not use. + DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem" + // DefaultClusterSigningKeyFile is deprecated. Do not use. + DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key" +) + // CSRSigningControllerOptions holds the CSRSigningController options. type CSRSigningControllerOptions struct { ClusterSigningDuration metav1.Duration diff --git a/cmd/controller-manager/app/options/generic.go b/cmd/controller-manager/app/options/generic.go index df9b19f8a39..d32b975691a 100644 --- a/cmd/controller-manager/app/options/generic.go +++ b/cmd/controller-manager/app/options/generic.go @@ -33,6 +33,22 @@ type GenericComponentConfigOptions struct { LeaderElection componentconfig.LeaderElectionConfiguration } +// NewGenericComponentConfigOptions returns generic configuration default values for both +// the kube-controller-manager and the cloud-contoller-manager. Any common changes should +// be made here. Any individual changes should be made in that controller. +func NewGenericComponentConfigOptions(cfg componentconfig.GenericComponentConfiguration) *GenericComponentConfigOptions { + o := &GenericComponentConfigOptions{ + MinResyncPeriod: cfg.MinResyncPeriod, + ContentType: cfg.ContentType, + KubeAPIQPS: cfg.KubeAPIQPS, + KubeAPIBurst: cfg.KubeAPIBurst, + ControllerStartInterval: cfg.ControllerStartInterval, + LeaderElection: cfg.LeaderElection, + } + + return o +} + // AddFlags adds flags related to generic for controller manager to the specified FlagSet. func (o *GenericComponentConfigOptions) AddFlags(fs *pflag.FlagSet) { if o == nil { diff --git a/cmd/controller-manager/app/options/kubecloudshared.go b/cmd/controller-manager/app/options/kubecloudshared.go index 823500b3644..d9607bf54b5 100644 --- a/cmd/controller-manager/app/options/kubecloudshared.go +++ b/cmd/controller-manager/app/options/kubecloudshared.go @@ -40,6 +40,22 @@ type KubeCloudSharedOptions struct { NodeSyncPeriod metav1.Duration } +// NewKubeCloudSharedOptions returns common/default configuration values for both +// the kube-controller-manager and the cloud-contoller-manager. Any common changes should +// be made here. Any individual changes should be made in that controller. +func NewKubeCloudSharedOptions(cfg componentconfig.KubeCloudSharedConfiguration) *KubeCloudSharedOptions { + o := &KubeCloudSharedOptions{ + Port: cfg.Port, + Address: cfg.Address, + RouteReconciliationPeriod: cfg.RouteReconciliationPeriod, + NodeMonitorPeriod: cfg.NodeMonitorPeriod, + ClusterName: cfg.ClusterName, + ConfigureCloudRoutes: cfg.ConfigureCloudRoutes, + } + + return o +} + // AddFlags adds flags related to shared variable for controller manager to the specified FlagSet. func (o *KubeCloudSharedOptions) AddFlags(fs *pflag.FlagSet) { if o == nil { diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go deleted file mode 100644 index 567a795b87b..00000000000 --- a/cmd/controller-manager/app/options/options.go +++ /dev/null @@ -1,383 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "net" - - "github.com/golang/glog" - - "github.com/spf13/pflag" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/client-go/kubernetes" - clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/record" - genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/apis/componentconfig" - componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" -) - -// GenericControllerManagerOptions is the common structure for a controller manager. It works with NewGenericControllerManagerOptions -// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager. -type GenericControllerManagerOptions struct { - CloudProvider *CloudProviderOptions - Debugging *DebuggingOptions - GenericComponent *GenericComponentConfigOptions - KubeCloudShared *KubeCloudSharedOptions - - AttachDetachController *AttachDetachControllerOptions - CSRSigningController *CSRSigningControllerOptions - DaemonSetController *DaemonSetControllerOptions - DeploymentController *DeploymentControllerOptions - DeprecatedFlags *DeprecatedControllerOptions - EndPointController *EndPointControllerOptions - GarbageCollectorController *GarbageCollectorControllerOptions - HPAController *HPAControllerOptions - JobController *JobControllerOptions - NamespaceController *NamespaceControllerOptions - NodeIpamController *NodeIpamControllerOptions - NodeLifecycleController *NodeLifecycleControllerOptions - PersistentVolumeBinderController *PersistentVolumeBinderControllerOptions - PodGCController *PodGCControllerOptions - ReplicaSetController *ReplicaSetControllerOptions - ReplicationController *ReplicationControllerOptions - ResourceQuotaController *ResourceQuotaControllerOptions - SAController *SAControllerOptions - ServiceController *ServiceControllerOptions - - Controllers []string - ExternalCloudVolumePlugin string - - SecureServing *apiserveroptions.SecureServingOptions - // TODO: remove insecure serving mode - InsecureServing *InsecureServingOptions - Authentication *apiserveroptions.DelegatingAuthenticationOptions - Authorization *apiserveroptions.DelegatingAuthorizationOptions - - Master string - Kubeconfig string -} - -const ( - // These defaults are deprecated and exported so that we can warn if - // they are being used. - - // DefaultClusterSigningCertFile is deprecated. Do not use. - DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem" - // DefaultClusterSigningKeyFile is deprecated. Do not use. - DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key" -) - -// NewGenericControllerManagerOptions returns common/default configuration values for both -// the kube-controller-manager and the cloud-contoller-manager. Any common changes should -// be made here. Any individual changes should be made in that controller. -func NewGenericControllerManagerOptions(componentConfig componentconfig.KubeControllerManagerConfiguration) *GenericControllerManagerOptions { - o := &GenericControllerManagerOptions{ - CloudProvider: &CloudProviderOptions{}, - Debugging: &DebuggingOptions{}, - GenericComponent: &GenericComponentConfigOptions{ - MinResyncPeriod: componentConfig.GenericComponent.MinResyncPeriod, - ContentType: componentConfig.GenericComponent.ContentType, - KubeAPIQPS: componentConfig.GenericComponent.KubeAPIQPS, - KubeAPIBurst: componentConfig.GenericComponent.KubeAPIBurst, - ControllerStartInterval: componentConfig.GenericComponent.ControllerStartInterval, - LeaderElection: componentConfig.GenericComponent.LeaderElection, - }, - KubeCloudShared: &KubeCloudSharedOptions{ - Port: componentConfig.KubeCloudShared.Port, - Address: componentConfig.KubeCloudShared.Address, - RouteReconciliationPeriod: componentConfig.KubeCloudShared.RouteReconciliationPeriod, - NodeMonitorPeriod: componentConfig.KubeCloudShared.NodeMonitorPeriod, - ClusterName: componentConfig.KubeCloudShared.ClusterName, - ConfigureCloudRoutes: componentConfig.KubeCloudShared.ConfigureCloudRoutes, - }, - AttachDetachController: &AttachDetachControllerOptions{ - ReconcilerSyncLoopPeriod: componentConfig.AttachDetachController.ReconcilerSyncLoopPeriod, - }, - CSRSigningController: &CSRSigningControllerOptions{ - ClusterSigningCertFile: componentConfig.CSRSigningController.ClusterSigningCertFile, - ClusterSigningKeyFile: componentConfig.CSRSigningController.ClusterSigningKeyFile, - ClusterSigningDuration: componentConfig.CSRSigningController.ClusterSigningDuration, - }, - DaemonSetController: &DaemonSetControllerOptions{ - ConcurrentDaemonSetSyncs: componentConfig.DaemonSetController.ConcurrentDaemonSetSyncs, - }, - DeploymentController: &DeploymentControllerOptions{ - ConcurrentDeploymentSyncs: componentConfig.DeploymentController.ConcurrentDeploymentSyncs, - DeploymentControllerSyncPeriod: componentConfig.DeploymentController.DeploymentControllerSyncPeriod, - }, - DeprecatedFlags: &DeprecatedControllerOptions{ - RegisterRetryCount: componentConfig.DeprecatedController.RegisterRetryCount, - }, - EndPointController: &EndPointControllerOptions{ - ConcurrentEndpointSyncs: componentConfig.EndPointController.ConcurrentEndpointSyncs, - }, - GarbageCollectorController: &GarbageCollectorControllerOptions{ - ConcurrentGCSyncs: componentConfig.GarbageCollectorController.ConcurrentGCSyncs, - EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector, - }, - HPAController: &HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod, - HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow, - HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow, - HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance, - HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients, - }, - JobController: &JobControllerOptions{ - ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs, - }, - NamespaceController: &NamespaceControllerOptions{ - NamespaceSyncPeriod: componentConfig.NamespaceController.NamespaceSyncPeriod, - ConcurrentNamespaceSyncs: componentConfig.NamespaceController.ConcurrentNamespaceSyncs, - }, - NodeIpamController: &NodeIpamControllerOptions{ - NodeCIDRMaskSize: componentConfig.NodeIpamController.NodeCIDRMaskSize, - }, - NodeLifecycleController: &NodeLifecycleControllerOptions{ - EnableTaintManager: componentConfig.NodeLifecycleController.EnableTaintManager, - NodeMonitorGracePeriod: componentConfig.NodeLifecycleController.NodeMonitorGracePeriod, - NodeStartupGracePeriod: componentConfig.NodeLifecycleController.NodeStartupGracePeriod, - PodEvictionTimeout: componentConfig.NodeLifecycleController.PodEvictionTimeout, - }, - PersistentVolumeBinderController: &PersistentVolumeBinderControllerOptions{ - PVClaimBinderSyncPeriod: componentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod, - VolumeConfiguration: componentConfig.PersistentVolumeBinderController.VolumeConfiguration, - }, - PodGCController: &PodGCControllerOptions{ - TerminatedPodGCThreshold: componentConfig.PodGCController.TerminatedPodGCThreshold, - }, - ReplicaSetController: &ReplicaSetControllerOptions{ - ConcurrentRSSyncs: componentConfig.ReplicaSetController.ConcurrentRSSyncs, - }, - ReplicationController: &ReplicationControllerOptions{ - ConcurrentRCSyncs: componentConfig.ReplicationController.ConcurrentRCSyncs, - }, - ResourceQuotaController: &ResourceQuotaControllerOptions{ - ResourceQuotaSyncPeriod: componentConfig.ResourceQuotaController.ResourceQuotaSyncPeriod, - ConcurrentResourceQuotaSyncs: componentConfig.ResourceQuotaController.ConcurrentResourceQuotaSyncs, - }, - SAController: &SAControllerOptions{ - ConcurrentSATokenSyncs: componentConfig.SAController.ConcurrentSATokenSyncs, - }, - ServiceController: &ServiceControllerOptions{ - ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, - }, - Controllers: componentConfig.Controllers, - SecureServing: apiserveroptions.NewSecureServingOptions(), - InsecureServing: &InsecureServingOptions{ - BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), - BindPort: int(componentConfig.KubeCloudShared.Port), - BindNetwork: "tcp", - }, - Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() - Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() - } - - // disable secure serving for now - // TODO: enable HTTPS by default - o.SecureServing.BindPort = 0 - - return o -} - -// NewDefaultControllerManagerComponentConfig returns default kube-controller manager configuration object. -func NewDefaultControllerManagerComponentConfig(insecurePort int32) componentconfig.KubeControllerManagerConfiguration { - scheme := runtime.NewScheme() - componentconfigv1alpha1.AddToScheme(scheme) - versioned := componentconfigv1alpha1.KubeControllerManagerConfiguration{} - scheme.Default(&versioned) - internal := componentconfig.KubeControllerManagerConfiguration{} - scheme.Convert(&versioned, &internal, nil) - internal.KubeCloudShared.Port = insecurePort - return internal -} - -// AddFlags adds common/default flags for both the kube and cloud Controller Manager Server to the -// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller. -func (o *GenericControllerManagerOptions) AddFlags(fs *pflag.FlagSet) { - - fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") - fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - o.CloudProvider.AddFlags(fs) - o.Debugging.AddFlags(fs) - o.GenericComponent.AddFlags(fs) - o.KubeCloudShared.AddFlags(fs) - o.ServiceController.AddFlags(fs) - o.SecureServing.AddFlags(fs) - o.InsecureServing.AddFlags(fs) - o.Authentication.AddFlags(fs) - o.Authorization.AddFlags(fs) -} - -// ApplyTo fills up controller manager config with options and userAgent -func (o *GenericControllerManagerOptions) ApplyTo(c *genericcontrollermanager.Config, userAgent string) error { - if err := o.CloudProvider.ApplyTo(&c.ComponentConfig.CloudProvider); err != nil { - return err - } - if err := o.Debugging.ApplyTo(&c.ComponentConfig.Debugging); err != nil { - return err - } - if err := o.GenericComponent.ApplyTo(&c.ComponentConfig.GenericComponent); err != nil { - return err - } - if err := o.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { - return err - } - if err := o.AttachDetachController.ApplyTo(&c.ComponentConfig.AttachDetachController); err != nil { - return err - } - if err := o.CSRSigningController.ApplyTo(&c.ComponentConfig.CSRSigningController); err != nil { - return err - } - if err := o.DaemonSetController.ApplyTo(&c.ComponentConfig.DaemonSetController); err != nil { - return err - } - if err := o.DeploymentController.ApplyTo(&c.ComponentConfig.DeploymentController); err != nil { - return err - } - if err := o.DeprecatedFlags.ApplyTo(&c.ComponentConfig.DeprecatedController); err != nil { - return err - } - if err := o.EndPointController.ApplyTo(&c.ComponentConfig.EndPointController); err != nil { - return err - } - if err := o.GarbageCollectorController.ApplyTo(&c.ComponentConfig.GarbageCollectorController); err != nil { - return err - } - if err := o.HPAController.ApplyTo(&c.ComponentConfig.HPAController); err != nil { - return err - } - if err := o.JobController.ApplyTo(&c.ComponentConfig.JobController); err != nil { - return err - } - if err := o.NamespaceController.ApplyTo(&c.ComponentConfig.NamespaceController); err != nil { - return err - } - if err := o.NodeIpamController.ApplyTo(&c.ComponentConfig.NodeIpamController); err != nil { - return err - } - if err := o.NodeLifecycleController.ApplyTo(&c.ComponentConfig.NodeLifecycleController); err != nil { - return err - } - if err := o.PersistentVolumeBinderController.ApplyTo(&c.ComponentConfig.PersistentVolumeBinderController); err != nil { - return err - } - if err := o.PodGCController.ApplyTo(&c.ComponentConfig.PodGCController); err != nil { - return err - } - if err := o.ReplicaSetController.ApplyTo(&c.ComponentConfig.ReplicaSetController); err != nil { - return err - } - if err := o.ReplicationController.ApplyTo(&c.ComponentConfig.ReplicationController); err != nil { - return err - } - if err := o.ResourceQuotaController.ApplyTo(&c.ComponentConfig.ResourceQuotaController); err != nil { - return err - } - if err := o.SAController.ApplyTo(&c.ComponentConfig.SAController); err != nil { - return err - } - if err := o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { - return err - } - if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil { - return err - } - if err := o.InsecureServing.ApplyTo(&c.InsecureServing); err != nil { - return err - } - if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err - } - if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { - return err - } - - // sync back to component config - // TODO: find more elegant way than synching back the values. - c.ComponentConfig.KubeCloudShared.Port = int32(o.InsecureServing.BindPort) - c.ComponentConfig.KubeCloudShared.Address = o.InsecureServing.BindAddress.String() - - var err error - c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig) - if err != nil { - return err - } - c.Kubeconfig.ContentConfig.ContentType = o.GenericComponent.ContentType - c.Kubeconfig.QPS = o.GenericComponent.KubeAPIQPS - c.Kubeconfig.Burst = int(o.GenericComponent.KubeAPIBurst) - - c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent)) - if err != nil { - return err - } - - c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election")) - - c.EventRecorder = createRecorder(c.Client, userAgent) - - return nil -} - -// Validate checks GenericControllerManagerOptions and return a slice of found errors. -func (o *GenericControllerManagerOptions) Validate() []error { - errors := []error{} - errors = append(errors, o.CloudProvider.Validate()...) - errors = append(errors, o.Debugging.Validate()...) - errors = append(errors, o.GenericComponent.Validate()...) - errors = append(errors, o.KubeCloudShared.Validate()...) - errors = append(errors, o.AttachDetachController.Validate()...) - errors = append(errors, o.CSRSigningController.Validate()...) - errors = append(errors, o.DaemonSetController.Validate()...) - errors = append(errors, o.DeploymentController.Validate()...) - errors = append(errors, o.DeprecatedFlags.Validate()...) - errors = append(errors, o.EndPointController.Validate()...) - errors = append(errors, o.GarbageCollectorController.Validate()...) - errors = append(errors, o.HPAController.Validate()...) - errors = append(errors, o.JobController.Validate()...) - errors = append(errors, o.NamespaceController.Validate()...) - errors = append(errors, o.NodeIpamController.Validate()...) - errors = append(errors, o.NodeLifecycleController.Validate()...) - errors = append(errors, o.PersistentVolumeBinderController.Validate()...) - errors = append(errors, o.PodGCController.Validate()...) - errors = append(errors, o.ReplicaSetController.Validate()...) - errors = append(errors, o.ReplicationController.Validate()...) - errors = append(errors, o.ResourceQuotaController.Validate()...) - errors = append(errors, o.SAController.Validate()...) - errors = append(errors, o.ServiceController.Validate()...) - errors = append(errors, o.SecureServing.Validate()...) - errors = append(errors, o.InsecureServing.Validate()...) - errors = append(errors, o.Authentication.Validate()...) - errors = append(errors, o.Authorization.Validate()...) - - // TODO: validate component config, master and kubeconfig - - return errors -} - -func createRecorder(kubeClient kubernetes.Interface, userAgent string) record.EventRecorder { - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) - return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent}) -} diff --git a/cmd/controller-manager/app/serve.go b/cmd/controller-manager/app/serve.go index 0b7251067da..305a3ba9d3b 100644 --- a/cmd/controller-manager/app/serve.go +++ b/cmd/controller-manager/app/serve.go @@ -24,21 +24,23 @@ import ( genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" apirequest "k8s.io/apiserver/pkg/endpoints/request" + apiserver "k8s.io/apiserver/pkg/server" genericfilters "k8s.io/apiserver/pkg/server/filters" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/routes" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/util/configz" ) // BuildHandlerChain builds a handler chain with a base handler and CompletedConfig. -func BuildHandlerChain(apiHandler http.Handler, c *CompletedConfig) http.Handler { +func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.AuthorizationInfo, authenticationInfo *apiserver.AuthenticationInfo) http.Handler { requestInfoResolver := &apirequest.RequestInfoFactory{} failedHandler := genericapifilters.Unauthorized(legacyscheme.Codecs, false) - handler := genericapifilters.WithAuthorization(apiHandler, c.Authorization.Authorizer, legacyscheme.Codecs) - handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler) + handler := genericapifilters.WithAuthorization(apiHandler, authorizationInfo.Authorizer, legacyscheme.Codecs) + handler = genericapifilters.WithAuthentication(handler, authenticationInfo.Authenticator, failedHandler) handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) handler = genericfilters.WithPanicRecovery(handler) @@ -46,12 +48,12 @@ func BuildHandlerChain(apiHandler http.Handler, c *CompletedConfig) http.Handler } // NewBaseHandler takes in CompletedConfig and returns a handler. -func NewBaseHandler(c *CompletedConfig) http.Handler { +func NewBaseHandler(c *componentconfig.DebuggingConfiguration) http.Handler { mux := mux.NewPathRecorderMux("controller-manager") healthz.InstallHandler(mux) - if c.ComponentConfig.Debugging.EnableProfiling { + if c.EnableProfiling { routes.Profiling{}.Install(mux) - if c.ComponentConfig.Debugging.EnableContentionProfiling { + if c.EnableContentionProfiling { goruntime.SetBlockProfileRate(1) } } diff --git a/cmd/kube-controller-manager/app/config/BUILD b/cmd/kube-controller-manager/app/config/BUILD index a9c61b0abad..73a13e94f17 100644 --- a/cmd/kube-controller-manager/app/config/BUILD +++ b/cmd/kube-controller-manager/app/config/BUILD @@ -5,7 +5,14 @@ go_library( srcs = ["config.go"], importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/config", visibility = ["//visibility:public"], - deps = ["//cmd/controller-manager/app:go_default_library"], + deps = [ + "//cmd/controller-manager/app:go_default_library", + "//pkg/apis/componentconfig:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + ], ) filegroup( diff --git a/cmd/kube-controller-manager/app/config/config.go b/cmd/kube-controller-manager/app/config/config.go index 4eb9c3ff812..4713e9dfe2e 100644 --- a/cmd/kube-controller-manager/app/config/config.go +++ b/cmd/kube-controller-manager/app/config/config.go @@ -17,25 +17,39 @@ limitations under the License. package config import ( - "time" - + apiserver "k8s.io/apiserver/pkg/server" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app" + "k8s.io/kubernetes/pkg/apis/componentconfig" ) -// ExtraConfig are part of Config, also can place your custom config here. -type ExtraConfig struct { - NodeStatusUpdateFrequency time.Duration -} - // Config is the main context object for the controller manager. type Config struct { - Generic genericcontrollermanager.Config - Extra ExtraConfig + ComponentConfig componentconfig.KubeControllerManagerConfiguration + + SecureServing *apiserver.SecureServingInfo + // TODO: remove deprecated insecure serving + InsecureServing *genericcontrollermanager.InsecureServingInfo + Authentication apiserver.AuthenticationInfo + Authorization apiserver.AuthorizationInfo + + // the general kube client + Client *clientset.Clientset + + // the client only used for leader election + LeaderElectionClient *clientset.Clientset + + // the rest config for the master + Kubeconfig *restclient.Config + + // the event sink + EventRecorder record.EventRecorder } type completedConfig struct { - Generic genericcontrollermanager.CompletedConfig - Extra *ExtraConfig + *Config } // CompletedConfig same as Config, just to swap private object. @@ -46,10 +60,6 @@ type CompletedConfig struct { // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() *CompletedConfig { - cc := completedConfig{ - c.Generic.Complete(), - &c.Extra, - } - + cc := completedConfig{c} return &CompletedConfig{&cc} } diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 9830be09690..b03ba75667e 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -108,7 +108,7 @@ controller, and serviceaccounts controller.`, func ResyncPeriod(c *config.CompletedConfig) func() time.Duration { return func() time.Duration { factor := rand.Float64() + 1 - return time.Duration(float64(c.Generic.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) + return time.Duration(float64(c.ComponentConfig.GenericComponent.MinResyncPeriod.Nanoseconds()) * factor) } } @@ -118,43 +118,43 @@ func Run(c *config.CompletedConfig) error { glog.Infof("Version: %+v", version.Get()) if cfgz, err := configz.New("componentconfig"); err == nil { - cfgz.Set(c.Generic.ComponentConfig) + cfgz.Set(c.ComponentConfig) } else { glog.Errorf("unable to register configz: %c", err) } // Start the controller manager HTTP server stopCh := make(chan struct{}) - if c.Generic.SecureServing != nil { - handler := genericcontrollermanager.NewBaseHandler(&c.Generic) - handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Generic) - if err := c.Generic.SecureServing.Serve(handler, 0, stopCh); err != nil { + if c.SecureServing != nil { + handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication) + if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } - if c.Generic.InsecureServing != nil { - handler := genericcontrollermanager.NewBaseHandler(&c.Generic) - handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Generic) - if err := c.Generic.InsecureServing.Serve(handler, 0, stopCh); err != nil { + if c.InsecureServing != nil { + handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) + handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication) + if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { return err } } run := func(stop <-chan struct{}) { rootClientBuilder := controller.SimpleControllerClientBuilder{ - ClientConfig: c.Generic.Kubeconfig, + ClientConfig: c.Kubeconfig, } var clientBuilder controller.ControllerClientBuilder - if c.Generic.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials { - if len(c.Generic.ComponentConfig.KubeCloudShared.ServiceAccountKeyFile) == 0 { + if c.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials { + if len(c.ComponentConfig.KubeCloudShared.ServiceAccountKeyFile) == 0 { // It'c possible another controller process is creating the tokens for us. // If one isn't, we'll timeout and exit when our client builder is unable to create the tokens. glog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file") } clientBuilder = controller.SAControllerClientBuilder{ - ClientConfig: restclient.AnonymousClientConfig(c.Generic.Kubeconfig), - CoreClient: c.Generic.Client.CoreV1(), - AuthenticationClient: c.Generic.Client.AuthenticationV1(), + ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), + CoreClient: c.Client.CoreV1(), + AuthenticationClient: c.Client.AuthenticationV1(), Namespace: "kube-system", } } else { @@ -176,7 +176,7 @@ func Run(c *config.CompletedConfig) error { select {} } - if !c.Generic.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { + if !c.ComponentConfig.GenericComponent.LeaderElection.LeaderElect { run(wait.NeverStop) panic("unreachable") } @@ -188,13 +188,13 @@ func Run(c *config.CompletedConfig) error { // add a uniquifier so that two processes on the same host don't accidentally both become active id = id + "_" + string(uuid.NewUUID()) - rl, err := resourcelock.New(c.Generic.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, + rl, err := resourcelock.New(c.ComponentConfig.GenericComponent.LeaderElection.ResourceLock, "kube-system", "kube-controller-manager", - c.Generic.LeaderElectionClient.CoreV1(), + c.LeaderElectionClient.CoreV1(), resourcelock.ResourceLockConfig{ Identity: id, - EventRecorder: c.Generic.EventRecorder, + EventRecorder: c.EventRecorder, }) if err != nil { glog.Fatalf("error creating lock: %v", err) @@ -202,9 +202,9 @@ func Run(c *config.CompletedConfig) error { leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ Lock: rl, - LeaseDuration: c.Generic.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, - RenewDeadline: c.Generic.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, - RetryPeriod: c.Generic.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, + LeaseDuration: c.ComponentConfig.GenericComponent.LeaderElection.LeaseDuration.Duration, + RenewDeadline: c.ComponentConfig.GenericComponent.LeaderElection.RenewDeadline.Duration, + RetryPeriod: c.ComponentConfig.GenericComponent.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { @@ -409,8 +409,8 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien return ControllerContext{}, err } - cloud, loopMode, err := createCloudProvider(s.Generic.ComponentConfig.CloudProvider.Name, s.Generic.ComponentConfig.ExternalCloudVolumePlugin, - s.Generic.ComponentConfig.CloudProvider.CloudConfigFile, s.Generic.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers) + cloud, loopMode, err := createCloudProvider(s.ComponentConfig.CloudProvider.Name, s.ComponentConfig.ExternalCloudVolumePlugin, + s.ComponentConfig.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers) if err != nil { return ControllerContext{}, err } @@ -418,7 +418,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien ctx := ControllerContext{ ClientBuilder: clientBuilder, InformerFactory: sharedInformers, - ComponentConfig: s.Generic.ComponentConfig, + ComponentConfig: s.ComponentConfig, RESTMapper: restMapper, AvailableResources: availableResources, Cloud: cloud, diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index 70d887cc7b0..3bba70af884 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -13,14 +13,25 @@ go_library( deps = [ "//cmd/controller-manager/app/options:go_default_library", "//cmd/kube-controller-manager/app/config:go_default_library", + "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/componentconfig:go_default_library", + "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/controller/garbagecollector:go_default_library", "//pkg/features:go_default_library", "//pkg/master/ports:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index b4bb98a79cb..2841dba49e4 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -20,77 +20,236 @@ package options import ( "fmt" + "net" "strings" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/componentconfig" + componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/controller/garbagecollector" "k8s.io/kubernetes/pkg/master/ports" - // add the kubernetes feature gates _ "k8s.io/kubernetes/pkg/features" + "github.com/golang/glog" "github.com/spf13/pflag" ) -// KubeControllerManagerOptions is the main context object for the controller manager. +// KubeControllerManagerOptions is the main context object for the kube-controller manager. type KubeControllerManagerOptions struct { - Generic *cmoptions.GenericControllerManagerOptions + CloudProvider *cmoptions.CloudProviderOptions + Debugging *cmoptions.DebuggingOptions + GenericComponent *cmoptions.GenericComponentConfigOptions + KubeCloudShared *cmoptions.KubeCloudSharedOptions + + AttachDetachController *cmoptions.AttachDetachControllerOptions + CSRSigningController *cmoptions.CSRSigningControllerOptions + DaemonSetController *cmoptions.DaemonSetControllerOptions + DeploymentController *cmoptions.DeploymentControllerOptions + DeprecatedFlags *cmoptions.DeprecatedControllerOptions + EndPointController *cmoptions.EndPointControllerOptions + GarbageCollectorController *cmoptions.GarbageCollectorControllerOptions + HPAController *cmoptions.HPAControllerOptions + JobController *cmoptions.JobControllerOptions + NamespaceController *cmoptions.NamespaceControllerOptions + NodeIpamController *cmoptions.NodeIpamControllerOptions + NodeLifecycleController *cmoptions.NodeLifecycleControllerOptions + PersistentVolumeBinderController *cmoptions.PersistentVolumeBinderControllerOptions + PodGCController *cmoptions.PodGCControllerOptions + ReplicaSetController *cmoptions.ReplicaSetControllerOptions + ReplicationController *cmoptions.ReplicationControllerOptions + ResourceQuotaController *cmoptions.ResourceQuotaControllerOptions + SAController *cmoptions.SAControllerOptions + ServiceController *cmoptions.ServiceControllerOptions + + Controllers []string + ExternalCloudVolumePlugin string + + SecureServing *apiserveroptions.SecureServingOptions + // TODO: remove insecure serving mode + InsecureServing *cmoptions.InsecureServingOptions + Authentication *apiserveroptions.DelegatingAuthenticationOptions + Authorization *apiserveroptions.DelegatingAuthorizationOptions + + Master string + Kubeconfig string } // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config. func NewKubeControllerManagerOptions() *KubeControllerManagerOptions { - componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureKubeControllerManagerPort) + componentConfig := NewDefaultComponentConfig(ports.InsecureKubeControllerManagerPort) s := KubeControllerManagerOptions{ - // The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'. - // Please make common changes there but put anything kube-controller specific here. - Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig), + CloudProvider: &cmoptions.CloudProviderOptions{}, + Debugging: &cmoptions.DebuggingOptions{}, + GenericComponent: cmoptions.NewGenericComponentConfigOptions(componentConfig.GenericComponent), + KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(componentConfig.KubeCloudShared), + AttachDetachController: &cmoptions.AttachDetachControllerOptions{ + ReconcilerSyncLoopPeriod: componentConfig.AttachDetachController.ReconcilerSyncLoopPeriod, + }, + CSRSigningController: &cmoptions.CSRSigningControllerOptions{ + ClusterSigningCertFile: componentConfig.CSRSigningController.ClusterSigningCertFile, + ClusterSigningKeyFile: componentConfig.CSRSigningController.ClusterSigningKeyFile, + ClusterSigningDuration: componentConfig.CSRSigningController.ClusterSigningDuration, + }, + DaemonSetController: &cmoptions.DaemonSetControllerOptions{ + ConcurrentDaemonSetSyncs: componentConfig.DaemonSetController.ConcurrentDaemonSetSyncs, + }, + DeploymentController: &cmoptions.DeploymentControllerOptions{ + ConcurrentDeploymentSyncs: componentConfig.DeploymentController.ConcurrentDeploymentSyncs, + DeploymentControllerSyncPeriod: componentConfig.DeploymentController.DeploymentControllerSyncPeriod, + }, + DeprecatedFlags: &cmoptions.DeprecatedControllerOptions{ + RegisterRetryCount: componentConfig.DeprecatedController.RegisterRetryCount, + }, + EndPointController: &cmoptions.EndPointControllerOptions{ + ConcurrentEndpointSyncs: componentConfig.EndPointController.ConcurrentEndpointSyncs, + }, + GarbageCollectorController: &cmoptions.GarbageCollectorControllerOptions{ + ConcurrentGCSyncs: componentConfig.GarbageCollectorController.ConcurrentGCSyncs, + EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector, + }, + HPAController: &cmoptions.HPAControllerOptions{ + HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod, + HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow, + HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow, + HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance, + HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients, + }, + JobController: &cmoptions.JobControllerOptions{ + ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs, + }, + NamespaceController: &cmoptions.NamespaceControllerOptions{ + NamespaceSyncPeriod: componentConfig.NamespaceController.NamespaceSyncPeriod, + ConcurrentNamespaceSyncs: componentConfig.NamespaceController.ConcurrentNamespaceSyncs, + }, + NodeIpamController: &cmoptions.NodeIpamControllerOptions{ + NodeCIDRMaskSize: componentConfig.NodeIpamController.NodeCIDRMaskSize, + }, + NodeLifecycleController: &cmoptions.NodeLifecycleControllerOptions{ + EnableTaintManager: componentConfig.NodeLifecycleController.EnableTaintManager, + NodeMonitorGracePeriod: componentConfig.NodeLifecycleController.NodeMonitorGracePeriod, + NodeStartupGracePeriod: componentConfig.NodeLifecycleController.NodeStartupGracePeriod, + PodEvictionTimeout: componentConfig.NodeLifecycleController.PodEvictionTimeout, + }, + PersistentVolumeBinderController: &cmoptions.PersistentVolumeBinderControllerOptions{ + PVClaimBinderSyncPeriod: componentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod, + VolumeConfiguration: componentConfig.PersistentVolumeBinderController.VolumeConfiguration, + }, + PodGCController: &cmoptions.PodGCControllerOptions{ + TerminatedPodGCThreshold: componentConfig.PodGCController.TerminatedPodGCThreshold, + }, + ReplicaSetController: &cmoptions.ReplicaSetControllerOptions{ + ConcurrentRSSyncs: componentConfig.ReplicaSetController.ConcurrentRSSyncs, + }, + ReplicationController: &cmoptions.ReplicationControllerOptions{ + ConcurrentRCSyncs: componentConfig.ReplicationController.ConcurrentRCSyncs, + }, + ResourceQuotaController: &cmoptions.ResourceQuotaControllerOptions{ + ResourceQuotaSyncPeriod: componentConfig.ResourceQuotaController.ResourceQuotaSyncPeriod, + ConcurrentResourceQuotaSyncs: componentConfig.ResourceQuotaController.ConcurrentResourceQuotaSyncs, + }, + SAController: &cmoptions.SAControllerOptions{ + ConcurrentSATokenSyncs: componentConfig.SAController.ConcurrentSATokenSyncs, + }, + ServiceController: &cmoptions.ServiceControllerOptions{ + ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, + }, + Controllers: componentConfig.Controllers, + SecureServing: apiserveroptions.NewSecureServingOptions(), + InsecureServing: &cmoptions.InsecureServingOptions{ + BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), + BindPort: int(componentConfig.KubeCloudShared.Port), + BindNetwork: "tcp", + }, + Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() + Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() } - s.Generic.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" - s.Generic.SecureServing.ServerCert.PairName = "kube-controller-manager" + s.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" + s.SecureServing.ServerCert.PairName = "kube-controller-manager" + + // disable secure serving for now + // TODO: enable HTTPS by default + s.SecureServing.BindPort = 0 gcIgnoredResources := make([]componentconfig.GroupResource, 0, len(garbagecollector.DefaultIgnoredResources())) for r := range garbagecollector.DefaultIgnoredResources() { gcIgnoredResources = append(gcIgnoredResources, componentconfig.GroupResource{Group: r.Group, Resource: r.Resource}) } - s.Generic.GarbageCollectorController.GCIgnoredResources = gcIgnoredResources + s.GarbageCollectorController.GCIgnoredResources = gcIgnoredResources return &s } +// NewDefaultComponentConfig returns kube-controller manager configuration object. +func NewDefaultComponentConfig(insecurePort int32) componentconfig.KubeControllerManagerConfiguration { + scheme := runtime.NewScheme() + componentconfigv1alpha1.AddToScheme(scheme) + componentconfig.AddToScheme(scheme) + + versioned := componentconfigv1alpha1.KubeControllerManagerConfiguration{} + scheme.Default(&versioned) + + internal := componentconfig.KubeControllerManagerConfiguration{} + scheme.Convert(&versioned, &internal, nil) + internal.KubeCloudShared.Port = insecurePort + return internal +} + // AddFlags adds flags for a specific KubeControllerManagerOptions to the specified FlagSet func (s *KubeControllerManagerOptions) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) { - s.Generic.AddFlags(fs) - s.Generic.AttachDetachController.AddFlags(fs) - s.Generic.CSRSigningController.AddFlags(fs) - s.Generic.DeploymentController.AddFlags(fs) - s.Generic.DaemonSetController.AddFlags(fs) - s.Generic.DeprecatedFlags.AddFlags(fs) - s.Generic.EndPointController.AddFlags(fs) - s.Generic.GarbageCollectorController.AddFlags(fs) - s.Generic.HPAController.AddFlags(fs) - s.Generic.JobController.AddFlags(fs) - s.Generic.NamespaceController.AddFlags(fs) - s.Generic.NodeIpamController.AddFlags(fs) - s.Generic.NodeLifecycleController.AddFlags(fs) - s.Generic.PersistentVolumeBinderController.AddFlags(fs) - s.Generic.PodGCController.AddFlags(fs) - s.Generic.ReplicaSetController.AddFlags(fs) - s.Generic.ReplicationController.AddFlags(fs) - s.Generic.ResourceQuotaController.AddFlags(fs) - s.Generic.SAController.AddFlags(fs) + s.CloudProvider.AddFlags(fs) + s.Debugging.AddFlags(fs) + s.GenericComponent.AddFlags(fs) + s.KubeCloudShared.AddFlags(fs) + s.ServiceController.AddFlags(fs) - fs.StringSliceVar(&s.Generic.Controllers, "controllers", s.Generic.Controllers, fmt.Sprintf(""+ + s.SecureServing.AddFlags(fs) + s.InsecureServing.AddFlags(fs) + s.Authentication.AddFlags(fs) + s.Authorization.AddFlags(fs) + + s.AttachDetachController.AddFlags(fs) + s.CSRSigningController.AddFlags(fs) + s.DeploymentController.AddFlags(fs) + s.DaemonSetController.AddFlags(fs) + s.DeprecatedFlags.AddFlags(fs) + s.EndPointController.AddFlags(fs) + s.GarbageCollectorController.AddFlags(fs) + s.HPAController.AddFlags(fs) + s.JobController.AddFlags(fs) + s.NamespaceController.AddFlags(fs) + s.NodeIpamController.AddFlags(fs) + s.NodeLifecycleController.AddFlags(fs) + s.PersistentVolumeBinderController.AddFlags(fs) + s.PodGCController.AddFlags(fs) + s.ReplicaSetController.AddFlags(fs) + s.ReplicationController.AddFlags(fs) + s.ResourceQuotaController.AddFlags(fs) + s.SAController.AddFlags(fs) + + fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") + fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") + fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+ "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) - fs.StringVar(&s.Generic.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.Generic.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.") + fs.StringVar(&s.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.") var dummy string fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.") fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.") @@ -98,11 +257,114 @@ func (s *KubeControllerManagerOptions) AddFlags(fs *pflag.FlagSet, allController } // ApplyTo fills up controller manager config with options. -func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) error { - err := s.Generic.ApplyTo(&c.Generic, "controller-manager") +func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config, userAgent string) error { + if err := s.CloudProvider.ApplyTo(&c.ComponentConfig.CloudProvider); err != nil { + return err + } + if err := s.Debugging.ApplyTo(&c.ComponentConfig.Debugging); err != nil { + return err + } + if err := s.GenericComponent.ApplyTo(&c.ComponentConfig.GenericComponent); err != nil { + return err + } + if err := s.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { + return err + } + if err := s.AttachDetachController.ApplyTo(&c.ComponentConfig.AttachDetachController); err != nil { + return err + } + if err := s.CSRSigningController.ApplyTo(&c.ComponentConfig.CSRSigningController); err != nil { + return err + } + if err := s.DaemonSetController.ApplyTo(&c.ComponentConfig.DaemonSetController); err != nil { + return err + } + if err := s.DeploymentController.ApplyTo(&c.ComponentConfig.DeploymentController); err != nil { + return err + } + if err := s.DeprecatedFlags.ApplyTo(&c.ComponentConfig.DeprecatedController); err != nil { + return err + } + if err := s.EndPointController.ApplyTo(&c.ComponentConfig.EndPointController); err != nil { + return err + } + if err := s.GarbageCollectorController.ApplyTo(&c.ComponentConfig.GarbageCollectorController); err != nil { + return err + } + if err := s.HPAController.ApplyTo(&c.ComponentConfig.HPAController); err != nil { + return err + } + if err := s.JobController.ApplyTo(&c.ComponentConfig.JobController); err != nil { + return err + } + if err := s.NamespaceController.ApplyTo(&c.ComponentConfig.NamespaceController); err != nil { + return err + } + if err := s.NodeIpamController.ApplyTo(&c.ComponentConfig.NodeIpamController); err != nil { + return err + } + if err := s.NodeLifecycleController.ApplyTo(&c.ComponentConfig.NodeLifecycleController); err != nil { + return err + } + if err := s.PersistentVolumeBinderController.ApplyTo(&c.ComponentConfig.PersistentVolumeBinderController); err != nil { + return err + } + if err := s.PodGCController.ApplyTo(&c.ComponentConfig.PodGCController); err != nil { + return err + } + if err := s.ReplicaSetController.ApplyTo(&c.ComponentConfig.ReplicaSetController); err != nil { + return err + } + if err := s.ReplicationController.ApplyTo(&c.ComponentConfig.ReplicationController); err != nil { + return err + } + if err := s.ResourceQuotaController.ApplyTo(&c.ComponentConfig.ResourceQuotaController); err != nil { + return err + } + if err := s.SAController.ApplyTo(&c.ComponentConfig.SAController); err != nil { + return err + } + if err := s.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { + return err + } + if err := s.SecureServing.ApplyTo(&c.SecureServing); err != nil { + return err + } + if err := s.InsecureServing.ApplyTo(&c.InsecureServing); err != nil { + return err + } + if err := s.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { + return err + } + if err := s.Authorization.ApplyTo(&c.Authorization); err != nil { + return err + } - c.Generic.ComponentConfig.Controllers = s.Generic.Controllers - c.Generic.ComponentConfig.ExternalCloudVolumePlugin = s.Generic.ExternalCloudVolumePlugin + // sync back to component config + // TODO: find more elegant way than synching back the values. + c.ComponentConfig.KubeCloudShared.Port = int32(s.InsecureServing.BindPort) + c.ComponentConfig.KubeCloudShared.Address = s.InsecureServing.BindAddress.String() + + var err error + c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) + if err != nil { + return err + } + c.Kubeconfig.ContentConfig.ContentType = s.GenericComponent.ContentType + c.Kubeconfig.QPS = s.GenericComponent.KubeAPIQPS + c.Kubeconfig.Burst = int(s.GenericComponent.KubeAPIBurst) + + c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent)) + if err != nil { + return err + } + + c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election")) + + c.EventRecorder = createRecorder(c.Client, userAgent) + + c.ComponentConfig.Controllers = s.Controllers + c.ComponentConfig.ExternalCloudVolumePlugin = s.ExternalCloudVolumePlugin return err } @@ -111,8 +373,38 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e func (s *KubeControllerManagerOptions) Validate(allControllers []string, disabledByDefaultControllers []string) error { var errs []error + errs = append(errs, s.CloudProvider.Validate()...) + errs = append(errs, s.Debugging.Validate()...) + errs = append(errs, s.GenericComponent.Validate()...) + errs = append(errs, s.KubeCloudShared.Validate()...) + errs = append(errs, s.AttachDetachController.Validate()...) + errs = append(errs, s.CSRSigningController.Validate()...) + errs = append(errs, s.DaemonSetController.Validate()...) + errs = append(errs, s.DeploymentController.Validate()...) + errs = append(errs, s.DeprecatedFlags.Validate()...) + errs = append(errs, s.EndPointController.Validate()...) + errs = append(errs, s.GarbageCollectorController.Validate()...) + errs = append(errs, s.HPAController.Validate()...) + errs = append(errs, s.JobController.Validate()...) + errs = append(errs, s.NamespaceController.Validate()...) + errs = append(errs, s.NodeIpamController.Validate()...) + errs = append(errs, s.NodeLifecycleController.Validate()...) + errs = append(errs, s.PersistentVolumeBinderController.Validate()...) + errs = append(errs, s.PodGCController.Validate()...) + errs = append(errs, s.ReplicaSetController.Validate()...) + errs = append(errs, s.ReplicationController.Validate()...) + errs = append(errs, s.ResourceQuotaController.Validate()...) + errs = append(errs, s.SAController.Validate()...) + errs = append(errs, s.ServiceController.Validate()...) + errs = append(errs, s.SecureServing.Validate()...) + errs = append(errs, s.InsecureServing.Validate()...) + errs = append(errs, s.Authentication.Validate()...) + errs = append(errs, s.Authorization.Validate()...) + + // TODO: validate component config, master and kubeconfig + allControllersSet := sets.NewString(allControllers...) - for _, controller := range s.Generic.Controllers { + for _, controller := range s.Controllers { if controller == "*" { continue } @@ -135,9 +427,16 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy } c := &kubecontrollerconfig.Config{} - if err := s.ApplyTo(c); err != nil { + if err := s.ApplyTo(c, "kube-controller-manager"); err != nil { return nil, err } return c, nil } + +func createRecorder(kubeClient kubernetes.Interface, userAgent string) record.EventRecorder { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent}) +} diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 2b4a86ee8f4..0e69b91b857 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -113,169 +113,167 @@ func TestAddFlags(t *testing.T) { f.Parse(args) // Sort GCIgnoredResources because it's built from a map, which means the // insertion order is random. - sort.Sort(sortedGCIgnoredResources(s.Generic.GarbageCollectorController.GCIgnoredResources)) + sort.Sort(sortedGCIgnoredResources(s.GarbageCollectorController.GCIgnoredResources)) expected := &KubeControllerManagerOptions{ - Generic: &cmoptions.GenericControllerManagerOptions{ - CloudProvider: &cmoptions.CloudProviderOptions{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - Debugging: &cmoptions.DebuggingOptions{ - EnableProfiling: false, - EnableContentionProfiling: true, - }, - GenericComponent: &cmoptions.GenericComponentConfigOptions{ - MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, - ContentType: "application/json", - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - LeaderElection: componentconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, - }, - }, - KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ - Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + AllocateNodeCIDRs: true, - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - UseServiceAccountCredentials: true, - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, - ClusterName: "k8s", - ClusterCIDR: "1.2.3.4/24", - AllocateNodeCIDRs: true, - CIDRAllocatorType: "CloudAllocator", - ConfigureCloudRoutes: false, - ServiceAccountKeyFile: "/service-account-private-key", - }, - AttachDetachController: &cmoptions.AttachDetachControllerOptions{ - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, - DisableAttachDetachReconcilerSync: true, - }, - CSRSigningController: &cmoptions.CSRSigningControllerOptions{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - }, - DaemonSetController: &cmoptions.DaemonSetControllerOptions{ - ConcurrentDaemonSetSyncs: 2, - }, - DeploymentController: &cmoptions.DeploymentControllerOptions{ - ConcurrentDeploymentSyncs: 10, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - }, - DeprecatedFlags: &cmoptions.DeprecatedControllerOptions{ - DeletingPodsQPS: 0.1, - RegisterRetryCount: 10, - }, - EndPointController: &cmoptions.EndPointControllerOptions{ - ConcurrentEndpointSyncs: 10, - }, - GarbageCollectorController: &cmoptions.GarbageCollectorControllerOptions{ - ConcurrentGCSyncs: 30, - GCIgnoredResources: []componentconfig.GroupResource{ - {Group: "extensions", Resource: "replicationcontrollers"}, - {Group: "", Resource: "bindings"}, - {Group: "", Resource: "componentstatuses"}, - {Group: "", Resource: "events"}, - {Group: "authentication.k8s.io", Resource: "tokenreviews"}, - {Group: "authorization.k8s.io", Resource: "subjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"}, - {Group: "apiregistration.k8s.io", Resource: "apiservices"}, - {Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}, - }, - EnableGarbageCollector: false, - }, - HPAController: &cmoptions.HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, - }, - JobController: &cmoptions.JobControllerOptions{ - ConcurrentJobSyncs: 5, - }, - NamespaceController: &cmoptions.NamespaceControllerOptions{ - NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentNamespaceSyncs: 20, - }, - NodeIpamController: &cmoptions.NodeIpamControllerOptions{ - NodeCIDRMaskSize: 48, - }, - NodeLifecycleController: &cmoptions.NodeLifecycleControllerOptions{ - EnableTaintManager: false, - NodeEvictionRate: 0.2, - SecondaryNodeEvictionRate: 0.05, - NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, - LargeClusterSizeThreshold: 100, - UnhealthyZoneThreshold: 0.6, - }, - PersistentVolumeBinderController: &cmoptions.PersistentVolumeBinderControllerOptions{ - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - VolumeConfiguration: componentconfig.VolumeConfiguration{ - EnableDynamicProvisioning: false, - EnableHostPathProvisioning: true, - FlexVolumePluginDir: "/flex-volume-plugin", - PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 200, - IncrementTimeoutNFS: 45, - MinimumTimeoutHostPath: 45, - IncrementTimeoutHostPath: 45, - }, - }, - }, - PodGCController: &cmoptions.PodGCControllerOptions{ - TerminatedPodGCThreshold: 12000, - }, - ReplicaSetController: &cmoptions.ReplicaSetControllerOptions{ - ConcurrentRSSyncs: 10, - }, - ReplicationController: &cmoptions.ReplicationControllerOptions{ - ConcurrentRCSyncs: 10, - }, - ResourceQuotaController: &cmoptions.ResourceQuotaControllerOptions{ - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentResourceQuotaSyncs: 10, - }, - SAController: &cmoptions.SAControllerOptions{ - ConcurrentSATokenSyncs: 10, - }, - ServiceController: &cmoptions.ServiceControllerOptions{ - ConcurrentServiceSyncs: 2, - }, - Controllers: []string{"foo", "bar"}, - SecureServing: &apiserveroptions.SecureServingOptions{ - BindPort: 10001, - BindAddress: net.ParseIP("192.168.4.21"), - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "kube-controller-manager", - }, - HTTP2MaxStreamsPerConnection: 47, - }, - InsecureServing: &cmoptions.InsecureServingOptions{ - BindAddress: net.ParseIP("192.168.4.10"), - BindPort: int(10000), - BindNetwork: "tcp", - }, - Kubeconfig: "/kubeconfig", - Master: "192.168.4.20", + CloudProvider: &cmoptions.CloudProviderOptions{ + Name: "gce", + CloudConfigFile: "/cloud-config", }, + Debugging: &cmoptions.DebuggingOptions{ + EnableProfiling: false, + EnableContentionProfiling: true, + }, + GenericComponent: &cmoptions.GenericComponentConfigOptions{ + MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, + ContentType: "application/json", + KubeAPIQPS: 50.0, + KubeAPIBurst: 100, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + LeaderElection: componentconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + }, + KubeCloudShared: &cmoptions.KubeCloudSharedOptions{ + Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + AllocateNodeCIDRs: true, + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + UseServiceAccountCredentials: true, + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, + ClusterName: "k8s", + ClusterCIDR: "1.2.3.4/24", + AllocateNodeCIDRs: true, + CIDRAllocatorType: "CloudAllocator", + ConfigureCloudRoutes: false, + ServiceAccountKeyFile: "/service-account-private-key", + }, + AttachDetachController: &cmoptions.AttachDetachControllerOptions{ + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, + DisableAttachDetachReconcilerSync: true, + }, + CSRSigningController: &cmoptions.CSRSigningControllerOptions{ + ClusterSigningCertFile: "/cluster-signing-cert", + ClusterSigningKeyFile: "/cluster-signing-key", + ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, + }, + DaemonSetController: &cmoptions.DaemonSetControllerOptions{ + ConcurrentDaemonSetSyncs: 2, + }, + DeploymentController: &cmoptions.DeploymentControllerOptions{ + ConcurrentDeploymentSyncs: 10, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + }, + DeprecatedFlags: &cmoptions.DeprecatedControllerOptions{ + DeletingPodsQPS: 0.1, + RegisterRetryCount: 10, + }, + EndPointController: &cmoptions.EndPointControllerOptions{ + ConcurrentEndpointSyncs: 10, + }, + GarbageCollectorController: &cmoptions.GarbageCollectorControllerOptions{ + ConcurrentGCSyncs: 30, + GCIgnoredResources: []componentconfig.GroupResource{ + {Group: "extensions", Resource: "replicationcontrollers"}, + {Group: "", Resource: "bindings"}, + {Group: "", Resource: "componentstatuses"}, + {Group: "", Resource: "events"}, + {Group: "authentication.k8s.io", Resource: "tokenreviews"}, + {Group: "authorization.k8s.io", Resource: "subjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"}, + {Group: "apiregistration.k8s.io", Resource: "apiservices"}, + {Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}, + }, + EnableGarbageCollector: false, + }, + HPAController: &cmoptions.HPAControllerOptions{ + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, + HorizontalPodAutoscalerTolerance: 0.1, + HorizontalPodAutoscalerUseRESTClients: true, + }, + JobController: &cmoptions.JobControllerOptions{ + ConcurrentJobSyncs: 5, + }, + NamespaceController: &cmoptions.NamespaceControllerOptions{ + NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + ConcurrentNamespaceSyncs: 20, + }, + NodeIpamController: &cmoptions.NodeIpamControllerOptions{ + NodeCIDRMaskSize: 48, + }, + NodeLifecycleController: &cmoptions.NodeLifecycleControllerOptions{ + EnableTaintManager: false, + NodeEvictionRate: 0.2, + SecondaryNodeEvictionRate: 0.05, + NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, + LargeClusterSizeThreshold: 100, + UnhealthyZoneThreshold: 0.6, + }, + PersistentVolumeBinderController: &cmoptions.PersistentVolumeBinderControllerOptions{ + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + VolumeConfiguration: componentconfig.VolumeConfiguration{ + EnableDynamicProvisioning: false, + EnableHostPathProvisioning: true, + FlexVolumePluginDir: "/flex-volume-plugin", + PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ + MaximumRetry: 3, + MinimumTimeoutNFS: 200, + IncrementTimeoutNFS: 45, + MinimumTimeoutHostPath: 45, + IncrementTimeoutHostPath: 45, + }, + }, + }, + PodGCController: &cmoptions.PodGCControllerOptions{ + TerminatedPodGCThreshold: 12000, + }, + ReplicaSetController: &cmoptions.ReplicaSetControllerOptions{ + ConcurrentRSSyncs: 10, + }, + ReplicationController: &cmoptions.ReplicationControllerOptions{ + ConcurrentRCSyncs: 10, + }, + ResourceQuotaController: &cmoptions.ResourceQuotaControllerOptions{ + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + ConcurrentResourceQuotaSyncs: 10, + }, + SAController: &cmoptions.SAControllerOptions{ + ConcurrentSATokenSyncs: 10, + }, + ServiceController: &cmoptions.ServiceControllerOptions{ + ConcurrentServiceSyncs: 2, + }, + Controllers: []string{"foo", "bar"}, + SecureServing: &apiserveroptions.SecureServingOptions{ + BindPort: 10001, + BindAddress: net.ParseIP("192.168.4.21"), + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-controller-manager", + }, + HTTP2MaxStreamsPerConnection: 47, + }, + InsecureServing: &cmoptions.InsecureServingOptions{ + BindAddress: net.ParseIP("192.168.4.10"), + BindPort: int(10000), + BindNetwork: "tcp", + }, + Kubeconfig: "/kubeconfig", + Master: "192.168.4.20", } // Sort GCIgnoredResources because it's built from a map, which means the // insertion order is random. - sort.Sort(sortedGCIgnoredResources(expected.Generic.GarbageCollectorController.GCIgnoredResources)) + sort.Sort(sortedGCIgnoredResources(expected.GarbageCollectorController.GCIgnoredResources)) if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s)) diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index e7460df3773..7ca6a224445 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -244,6 +244,28 @@ type KubeControllerManagerConfiguration struct { ExternalCloudVolumePlugin string } +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type CloudControllerManagerConfiguration struct { + metav1.TypeMeta + + // CloudProviderConfiguration holds configuration for CloudProvider related features. + CloudProvider CloudProviderConfiguration + // DebuggingConfiguration holds configuration for Debugging related features. + Debugging DebuggingConfiguration + // GenericComponentConfiguration holds configuration for GenericComponent + // related features both in cloud controller manager and kube-controller manager. + GenericComponent GenericComponentConfiguration + // KubeCloudSharedConfiguration holds configuration for shared related features + // both in cloud controller manager and kube-controller manager. + KubeCloudShared KubeCloudSharedConfiguration + // ServiceControllerConfiguration holds configuration for ServiceController + // related features. + ServiceController ServiceControllerConfiguration + // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status + NodeStatusUpdateFrequency metav1.Duration +} + type CloudProviderConfiguration struct { // Name is the provider for cloud services. Name string diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 925412c7138..225b279a03c 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -33,15 +33,21 @@ func addDefaultingFuncs(scheme *kruntime.Scheme) error { return RegisterDefaults(scheme) } +func SetDefaults_CloudControllerManagerConfiguration(obj *CloudControllerManagerConfiguration) { + zero := metav1.Duration{} + if obj.ServiceController.ConcurrentServiceSyncs == 0 { + obj.ServiceController.ConcurrentServiceSyncs = 1 + } + if obj.NodeStatusUpdateFrequency == zero { + obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 5 * time.Minute} + } +} + func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerConfiguration) { zero := metav1.Duration{} if len(obj.Controllers) == 0 { obj.Controllers = []string{"*"} } - // Port - if obj.KubeCloudShared.Address == "" { - obj.KubeCloudShared.Address = "0.0.0.0" - } if obj.EndPointController.ConcurrentEndpointSyncs == 0 { obj.EndPointController.ConcurrentEndpointSyncs = 5 } @@ -72,9 +78,6 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo if obj.SAController.ConcurrentSATokenSyncs == 0 { obj.SAController.ConcurrentSATokenSyncs = 5 } - if obj.KubeCloudShared.RouteReconciliationPeriod == zero { - obj.KubeCloudShared.RouteReconciliationPeriod = metav1.Duration{Duration: 10 * time.Second} - } if obj.ResourceQuotaController.ResourceQuotaSyncPeriod == zero { obj.ResourceQuotaController.ResourceQuotaSyncPeriod = metav1.Duration{Duration: 5 * time.Minute} } @@ -99,9 +102,6 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo if obj.DeploymentController.DeploymentControllerSyncPeriod == zero { obj.DeploymentController.DeploymentControllerSyncPeriod = metav1.Duration{Duration: 30 * time.Second} } - if obj.GenericComponent.MinResyncPeriod == zero { - obj.GenericComponent.MinResyncPeriod = metav1.Duration{Duration: 12 * time.Hour} - } if obj.DeprecatedController.RegisterRetryCount == 0 { obj.DeprecatedController.RegisterRetryCount = 10 } @@ -114,33 +114,12 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo if obj.NodeLifecycleController.NodeStartupGracePeriod == zero { obj.NodeLifecycleController.NodeStartupGracePeriod = metav1.Duration{Duration: 60 * time.Second} } - if obj.KubeCloudShared.NodeMonitorPeriod == zero { - obj.KubeCloudShared.NodeMonitorPeriod = metav1.Duration{Duration: 5 * time.Second} - } - if obj.KubeCloudShared.ClusterName == "" { - obj.KubeCloudShared.ClusterName = "kubernetes" - } if obj.NodeIpamController.NodeCIDRMaskSize == 0 { obj.NodeIpamController.NodeCIDRMaskSize = 24 } - if obj.KubeCloudShared.ConfigureCloudRoutes == nil { - obj.KubeCloudShared.ConfigureCloudRoutes = utilpointer.BoolPtr(true) - } if obj.PodGCController.TerminatedPodGCThreshold == 0 { obj.PodGCController.TerminatedPodGCThreshold = 12500 } - if obj.GenericComponent.ContentType == "" { - obj.GenericComponent.ContentType = "application/vnd.kubernetes.protobuf" - } - if obj.GenericComponent.KubeAPIQPS == 0 { - obj.GenericComponent.KubeAPIQPS = 20.0 - } - if obj.GenericComponent.KubeAPIBurst == 0 { - obj.GenericComponent.KubeAPIBurst = 30 - } - if obj.GenericComponent.ControllerStartInterval == zero { - obj.GenericComponent.ControllerStartInterval = metav1.Duration{Duration: 0 * time.Second} - } if obj.GarbageCollectorController.EnableGarbageCollector == nil { obj.GarbageCollectorController.EnableGarbageCollector = utilpointer.BoolPtr(true) } @@ -167,6 +146,45 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo } } +func SetDefaults_GenericComponentConfiguration(obj *GenericComponentConfiguration) { + zero := metav1.Duration{} + if obj.MinResyncPeriod == zero { + obj.MinResyncPeriod = metav1.Duration{Duration: 12 * time.Hour} + } + if obj.ContentType == "" { + obj.ContentType = "application/vnd.kubernetes.protobuf" + } + if obj.KubeAPIQPS == 0 { + obj.KubeAPIQPS = 20.0 + } + if obj.KubeAPIBurst == 0 { + obj.KubeAPIBurst = 30 + } + if obj.ControllerStartInterval == zero { + obj.ControllerStartInterval = metav1.Duration{Duration: 0 * time.Second} + } +} + +func SetDefaults_KubeCloudSharedConfiguration(obj *KubeCloudSharedConfiguration) { + zero := metav1.Duration{} + // Port + if obj.Address == "" { + obj.Address = "0.0.0.0" + } + if obj.RouteReconciliationPeriod == zero { + obj.RouteReconciliationPeriod = metav1.Duration{Duration: 10 * time.Second} + } + if obj.NodeMonitorPeriod == zero { + obj.NodeMonitorPeriod = metav1.Duration{Duration: 5 * time.Second} + } + if obj.ClusterName == "" { + obj.ClusterName = "kubernetes" + } + if obj.ConfigureCloudRoutes == nil { + obj.ConfigureCloudRoutes = utilpointer.BoolPtr(true) + } +} + func SetDefaults_PersistentVolumeRecyclerConfiguration(obj *PersistentVolumeRecyclerConfiguration) { if obj.MaximumRetry == 0 { obj.MaximumRetry = 3 diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index ccf81061dc6..129f5d650c5 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -286,6 +286,28 @@ type KubeControllerManagerConfiguration struct { ExternalCloudVolumePlugin string `json:"externalCloudVolumePlugin"` } +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type CloudControllerManagerConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // CloudProviderConfiguration holds configuration for CloudProvider related features. + CloudProvider CloudProviderConfiguration + // DebuggingConfiguration holds configuration for Debugging related features. + Debugging DebuggingConfiguration + // GenericComponentConfiguration holds configuration for GenericComponent + // related features both in cloud controller manager and kube-controller manager. + GenericComponent GenericComponentConfiguration + // KubeCloudSharedConfiguration holds configuration for shared related features + // both in cloud controller manager and kube-controller manager. + KubeCloudShared KubeCloudSharedConfiguration + // ServiceControllerConfiguration holds configuration for ServiceController + // related features. + ServiceController ServiceControllerConfiguration + // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status + NodeStatusUpdateFrequency metav1.Duration +} + type CloudProviderConfiguration struct { // Name is the provider for cloud services. Name string `json:"cloudProvider"` diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index c4b56fc563b..9e6cc33cb32 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -43,6 +43,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_componentconfig_CSRSigningControllerConfiguration_To_v1alpha1_CSRSigningControllerConfiguration, Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration, Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration, + Convert_v1alpha1_CloudControllerManagerConfiguration_To_componentconfig_CloudControllerManagerConfiguration, + Convert_componentconfig_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration, Convert_v1alpha1_CloudProviderConfiguration_To_componentconfig_CloudProviderConfiguration, Convert_componentconfig_CloudProviderConfiguration_To_v1alpha1_CloudProviderConfiguration, Convert_v1alpha1_DaemonSetControllerConfiguration_To_componentconfig_DaemonSetControllerConfiguration, @@ -184,6 +186,56 @@ func Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientCon return autoConvert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(in, out, s) } +func autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_componentconfig_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration, out *componentconfig.CloudControllerManagerConfiguration, s conversion.Scope) error { + if err := Convert_v1alpha1_CloudProviderConfiguration_To_componentconfig_CloudProviderConfiguration(&in.CloudProvider, &out.CloudProvider, s); err != nil { + return err + } + if err := Convert_v1alpha1_DebuggingConfiguration_To_componentconfig_DebuggingConfiguration(&in.Debugging, &out.Debugging, s); err != nil { + return err + } + if err := Convert_v1alpha1_GenericComponentConfiguration_To_componentconfig_GenericComponentConfiguration(&in.GenericComponent, &out.GenericComponent, s); err != nil { + return err + } + if err := Convert_v1alpha1_KubeCloudSharedConfiguration_To_componentconfig_KubeCloudSharedConfiguration(&in.KubeCloudShared, &out.KubeCloudShared, s); err != nil { + return err + } + if err := Convert_v1alpha1_ServiceControllerConfiguration_To_componentconfig_ServiceControllerConfiguration(&in.ServiceController, &out.ServiceController, s); err != nil { + return err + } + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + return nil +} + +// Convert_v1alpha1_CloudControllerManagerConfiguration_To_componentconfig_CloudControllerManagerConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_CloudControllerManagerConfiguration_To_componentconfig_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration, out *componentconfig.CloudControllerManagerConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_componentconfig_CloudControllerManagerConfiguration(in, out, s) +} + +func autoConvert_componentconfig_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in *componentconfig.CloudControllerManagerConfiguration, out *CloudControllerManagerConfiguration, s conversion.Scope) error { + if err := Convert_componentconfig_CloudProviderConfiguration_To_v1alpha1_CloudProviderConfiguration(&in.CloudProvider, &out.CloudProvider, s); err != nil { + return err + } + if err := Convert_componentconfig_DebuggingConfiguration_To_v1alpha1_DebuggingConfiguration(&in.Debugging, &out.Debugging, s); err != nil { + return err + } + if err := Convert_componentconfig_GenericComponentConfiguration_To_v1alpha1_GenericComponentConfiguration(&in.GenericComponent, &out.GenericComponent, s); err != nil { + return err + } + if err := Convert_componentconfig_KubeCloudSharedConfiguration_To_v1alpha1_KubeCloudSharedConfiguration(&in.KubeCloudShared, &out.KubeCloudShared, s); err != nil { + return err + } + if err := Convert_componentconfig_ServiceControllerConfiguration_To_v1alpha1_ServiceControllerConfiguration(&in.ServiceController, &out.ServiceController, s); err != nil { + return err + } + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + return nil +} + +// Convert_componentconfig_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration is an autogenerated conversion function. +func Convert_componentconfig_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in *componentconfig.CloudControllerManagerConfiguration, out *CloudControllerManagerConfiguration, s conversion.Scope) error { + return autoConvert_componentconfig_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in, out, s) +} + func autoConvert_v1alpha1_CloudProviderConfiguration_To_componentconfig_CloudProviderConfiguration(in *CloudProviderConfiguration, out *componentconfig.CloudProviderConfiguration, s conversion.Scope) error { out.Name = in.Name out.CloudConfigFile = in.CloudConfigFile diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index 0103c1a6e3e..45cf5d9afe0 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -74,6 +74,37 @@ func (in *ClientConnectionConfiguration) DeepCopy() *ClientConnectionConfigurati return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudControllerManagerConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + out.CloudProvider = in.CloudProvider + out.Debugging = in.Debugging + in.GenericComponent.DeepCopyInto(&out.GenericComponent) + in.KubeCloudShared.DeepCopyInto(&out.KubeCloudShared) + out.ServiceController = in.ServiceController + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfiguration. +func (in *CloudControllerManagerConfiguration) DeepCopy() *CloudControllerManagerConfiguration { + if in == nil { + return nil + } + out := new(CloudControllerManagerConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudControllerManagerConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProviderConfiguration) DeepCopyInto(out *CloudProviderConfiguration) { *out = *in diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.defaults.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.defaults.go index 8c6ac4518be..75fd9b76226 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.defaults.go @@ -28,6 +28,9 @@ import ( // Public to allow building arbitrary schemes. // All generated defaulters are covering - they call all nested defaulters. func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&CloudControllerManagerConfiguration{}, func(obj interface{}) { + SetObjectDefaults_CloudControllerManagerConfiguration(obj.(*CloudControllerManagerConfiguration)) + }) scheme.AddTypeDefaultingFunc(&KubeControllerManagerConfiguration{}, func(obj interface{}) { SetObjectDefaults_KubeControllerManagerConfiguration(obj.(*KubeControllerManagerConfiguration)) }) @@ -35,9 +38,18 @@ func RegisterDefaults(scheme *runtime.Scheme) error { return nil } +func SetObjectDefaults_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration) { + SetDefaults_CloudControllerManagerConfiguration(in) + SetDefaults_GenericComponentConfiguration(&in.GenericComponent) + SetDefaults_LeaderElectionConfiguration(&in.GenericComponent.LeaderElection) + SetDefaults_KubeCloudSharedConfiguration(&in.KubeCloudShared) +} + func SetObjectDefaults_KubeControllerManagerConfiguration(in *KubeControllerManagerConfiguration) { SetDefaults_KubeControllerManagerConfiguration(in) + SetDefaults_GenericComponentConfiguration(&in.GenericComponent) SetDefaults_LeaderElectionConfiguration(&in.GenericComponent.LeaderElection) + SetDefaults_KubeCloudSharedConfiguration(&in.KubeCloudShared) SetDefaults_VolumeConfiguration(&in.PersistentVolumeBinderController.VolumeConfiguration) SetDefaults_PersistentVolumeRecyclerConfiguration(&in.PersistentVolumeBinderController.VolumeConfiguration.PersistentVolumeRecyclerConfiguration) } diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index cf3a2c295f0..c8317d119b3 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -74,6 +74,37 @@ func (in *ClientConnectionConfiguration) DeepCopy() *ClientConnectionConfigurati return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudControllerManagerConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + out.CloudProvider = in.CloudProvider + out.Debugging = in.Debugging + out.GenericComponent = in.GenericComponent + out.KubeCloudShared = in.KubeCloudShared + out.ServiceController = in.ServiceController + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfiguration. +func (in *CloudControllerManagerConfiguration) DeepCopy() *CloudControllerManagerConfiguration { + if in == nil { + return nil + } + out := new(CloudControllerManagerConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudControllerManagerConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProviderConfiguration) DeepCopyInto(out *CloudProviderConfiguration) { *out = *in