From b754393630362280ef13591813b1817eabc9f939 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 14 May 2016 09:50:17 -0400 Subject: [PATCH] kube-controller-manager: Add configure-cloud-routes option This allows kube-controller-manager to allocate CIDRs to nodes (with allocate-node-cidrs=true), but will not try to configure them on the cloud provider, even if the cloud provider supports Routes. The default is configure-cloud-routes=true, and it will only try to configure routes if allocate-node-cidrs is also configured, so the default behaviour is unchanged. This is useful because on AWS the cloud provider configures routes by setting up VPC routing table entries, but there is a limit of 50 entries. So setting configure-cloud-routes on AWS would allow us to continue to allocate node CIDRs as today, but replace the VPC route-table mechanism with something not limited to 50 nodes. We can't just turn off the cloud-provider entirely because it also controls other things - node discovery, load balancer creation etc. Fix #25602 --- cmd/kube-controller-manager/app/controllermanager.go | 12 +++++++----- cmd/kube-controller-manager/app/options/options.go | 2 ++ .../mesos/pkg/controllermanager/controllermanager.go | 4 ++-- docs/admin/kube-controller-manager.md | 1 + hack/verify-flags/known-flags.txt | 1 + pkg/apis/componentconfig/types.go | 7 +++++-- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index be6ae9a17cf..b97fb1b26d7 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -245,18 +245,20 @@ func StartControllers(s *options.CMServer, kubeClient *client.Client, kubeconfig } time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter)) - if s.AllocateNodeCIDRs { + if s.AllocateNodeCIDRs && s.ConfigureCloudRoutes { if cloud == nil { - glog.Warning("allocate-node-cidrs is set, but no cloud provider specified. Will not manage routes.") + glog.Warning("configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.") } else if routes, ok := cloud.Routes(); !ok { - glog.Warning("allocate-node-cidrs is set, but cloud provider does not support routes. Will not manage routes.") + glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") } else { routeController := routecontroller.New(routes, clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "route-controller")), s.ClusterName, clusterCIDR) routeController.Run(s.NodeSyncPeriod.Duration) time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter)) } - } else { - glog.Infof("allocate-node-cidrs set to %v, node controller not creating routes", s.AllocateNodeCIDRs) + } else if s.ConfigureCloudRoutes && !s.AllocateNodeCIDRs { + glog.Warningf("allocate-node-cidrs set to %v, will not configure cloud provider routes.", s.AllocateNodeCIDRs) + } else if s.AllocateNodeCIDRs && !s.ConfigureCloudRoutes { + glog.Infof("configure-cloud-routes is set to %v, will not configure cloud provider routes.", s.ConfigureCloudRoutes) } resourceQuotaControllerClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "resourcequota-controller")) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index ed6c9306f00..a87067fa43b 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -71,6 +71,7 @@ func NewCMServer() *CMServer { NodeMonitorPeriod: unversioned.Duration{Duration: 5 * time.Second}, ClusterName: "kubernetes", NodeCIDRMaskSize: 24, + ConfigureCloudRoutes: true, TerminatedPodGCThreshold: 12500, VolumeConfiguration: componentconfig.VolumeConfiguration{ EnableHostPathProvisioning: false, @@ -148,6 +149,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ServiceCIDR, "service-cluster-ip-range", s.ServiceCIDR, "CIDR Range for Services in cluster.") fs.Int32Var(&s.NodeCIDRMaskSize, "node-cidr-mask-size", s.NodeCIDRMaskSize, "Mask size for node cidr in cluster.") fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.") + fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") fs.StringVar(&s.RootCAFile, "root-ca-file", s.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.") diff --git a/contrib/mesos/pkg/controllermanager/controllermanager.go b/contrib/mesos/pkg/controllermanager/controllermanager.go index 7d843604f86..7c7f602a843 100644 --- a/contrib/mesos/pkg/controllermanager/controllermanager.go +++ b/contrib/mesos/pkg/controllermanager/controllermanager.go @@ -170,10 +170,10 @@ func (s *CMServer) Run(_ []string) error { glog.Errorf("Failed to start service controller: %v", err) } - if s.AllocateNodeCIDRs { + if s.AllocateNodeCIDRs && s.ConfigureCloudRoutes { routes, ok := cloud.Routes() if !ok { - glog.Fatal("Cloud provider must support routes if allocate-node-cidrs is set") + glog.Fatal("Cloud provider must support routes if configure-cloud-routes is set") } routeController := routecontroller.New(routes, clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "route-controller")), s.ClusterName, clusterCIDR) routeController.Run(s.NodeSyncPeriod.Duration) diff --git a/docs/admin/kube-controller-manager.md b/docs/admin/kube-controller-manager.md index 734e20dc878..ca47b1fe7fd 100644 --- a/docs/admin/kube-controller-manager.md +++ b/docs/admin/kube-controller-manager.md @@ -67,6 +67,7 @@ kube-controller-manager --concurrent-replicaset-syncs=5: The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load --concurrent-resource-quota-syncs=5: The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load --concurrent_rc_syncs=5: The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load + --configure-cloud-routes[=true]: Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider. --controller-start-interval=0: Interval between starting controller managers. --daemonset-lookup-cache-size=1024: The the size of lookup cache for daemonsets. Larger number = more responsive daemonsets, but more MEM load. --deleting-pods-burst=10: Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter. diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1591f9e7581..72ced7c0b36 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -66,6 +66,7 @@ concurrent-replicaset-syncs concurrent-resource-quota-syncs config-sync-period configure-cbr0 +configure-cloud-routes conntrack-max conntrack-tcp-timeout-established contain-pod-resources diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index f550a74cb26..98ded77e006 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -533,9 +533,12 @@ type KubeControllerManagerConfiguration struct { ServiceCIDR string `json:"serviceCIDR"` // NodeCIDRMaskSize is the mask size for node cidr in cluster. NodeCIDRMaskSize int32 `json:"nodeCIDRMaskSize"` - // allocateNodeCIDRs enables CIDRs for Pods to be allocated and set on the - // cloud provider. + // allocateNodeCIDRs enables CIDRs for Pods to be allocated and, if + // ConfigureCloudRoutes is true, to be set on the cloud provider. AllocateNodeCIDRs bool `json:"allocateNodeCIDRs"` + // configureCloudRoutes enables CIDRs allocated with allocateNodeCIDRs + // to be configured on the cloud provider. + ConfigureCloudRoutes bool `json:"configureCloudRoutes"` // rootCAFile is the root certificate authority will be included in service // account's token secret. This must be a valid PEM-encoded CA bundle. RootCAFile string `json:"rootCAFile"`