From 3a5ef05f64d22aa609445ffd76e5beb00b450b99 Mon Sep 17 00:00:00 2001 From: Ravi Sankar Penta Date: Wed, 14 Jan 2015 17:05:48 -0800 Subject: [PATCH] Make periodic sync nodes from cloud provider optional. --sync_nodes=false gives user flexibility to add/remove nodes in the cluster using REST api/kubectl cli and at the same time can use cloud provider for other resources like persistent disks, etc. --- .../saltbase/salt/kube-controller-manager/default | 3 ++- cmd/integration/integration.go | 2 +- cmd/kubernetes/kubernetes.go | 2 +- docs/man/kube-controller-manager.1.md | 5 ++++- docs/node.md | 6 ++++-- pkg/cloudprovider/controller/nodecontroller.go | 14 +++++++++----- pkg/controllermanager/controllermanager.go | 11 +++++++++-- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/cluster/saltbase/salt/kube-controller-manager/default b/cluster/saltbase/salt/kube-controller-manager/default index c5136546ebe..7dcdddba765 100644 --- a/cluster/saltbase/salt/kube-controller-manager/default +++ b/cluster/saltbase/salt/kube-controller-manager/default @@ -9,6 +9,7 @@ {% set machines = ""-%} {% set cloud_provider = "" -%} {% set minion_regexp = "--minion_regexp=.*" -%} +{% set sync_nodes = "--sync_nodes=true" -%} {% if grains.cloud_provider is defined -%} {% set cloud_provider = "--cloud_provider=" + grains.cloud_provider -%} {% endif -%} @@ -47,4 +48,4 @@ {% endif -%} {% endif -%} -DAEMON_ARGS="{{daemon_args}} {{master}} {{machines}} {{ minion_regexp }} {{ cloud_provider }} {{ cloud_config }} {{pillar['log_level']}}" +DAEMON_ARGS="{{daemon_args}} {{master}} {{machines}} {{ minion_regexp }} {{ cloud_provider }} {{ sync_nodes }} {{ cloud_config }} {{pillar['log_level']}}" diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index b4c47a3fcb8..5efb6f05e19 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -198,7 +198,7 @@ func startComponents(manifestURL string) (apiServerURL string) { nodeResources := &api.NodeResources{} nodeController := nodeControllerPkg.NewNodeController(nil, "", machineList, nodeResources, cl, fakeKubeletClient{}) - nodeController.Run(5*time.Second, 10) + nodeController.Run(5*time.Second, 10, true) // Kubelet (localhost) testRootDir := makeTempDirOrDie("kubelet_integ_1.") diff --git a/cmd/kubernetes/kubernetes.go b/cmd/kubernetes/kubernetes.go index e01fabd565e..a81209d750f 100644 --- a/cmd/kubernetes/kubernetes.go +++ b/cmd/kubernetes/kubernetes.go @@ -124,7 +124,7 @@ func runControllerManager(machineList []string, cl *client.Client, nodeMilliCPU, } kubeClient := &client.HTTPKubeletClient{Client: http.DefaultClient, Port: ports.KubeletPort} nodeController := nodeControllerPkg.NewNodeController(nil, "", machineList, nodeResources, cl, kubeClient) - nodeController.Run(10*time.Second, 10) + nodeController.Run(10*time.Second, 10, true) endpoints := service.NewEndpointController(cl) go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10) diff --git a/docs/man/kube-controller-manager.1.md b/docs/man/kube-controller-manager.1.md index 52a2d9ec564..14e73f81a7d 100644 --- a/docs/man/kube-controller-manager.1.md +++ b/docs/man/kube-controller-manager.1.md @@ -36,7 +36,7 @@ The kube-controller-manager has several options. The provider for cloud services. Empty string for no provider. **--minion_regexp**="" - If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs. + If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs. **--insecure_skip_tls_verify**=false If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure. @@ -56,6 +56,9 @@ The kube-controller-manager has several options. **--machines**=[] List of machines to schedule onto, comma separated. +**--sync_nodes**=true + If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true. + **--master**="" The address of the Kubernetes API server. diff --git a/docs/node.md b/docs/node.md index eed19d46f09..cf99e305cd5 100644 --- a/docs/node.md +++ b/docs/node.md @@ -110,9 +110,11 @@ any binary; therefore, to join Kubernetes cluster, you as an admin need to make sure proper services are running in the node. In the future, we plan to automatically provision some node services. In case of no cloud provider, Node Controller simply registers all -machines from `-machines` flag, any futher interactions need to be done manually -by using `kubectl`. If you are paranoid, leave `-machines` empty and create all +machines from `--machines` flag, any futher interactions need to be done manually +by using `kubectl`. If you are paranoid, leave `--machines` empty and create all machines from `kubectl` one by one - the two approaches are equivalent. +Optionally you can skip cluster-wide node synchronization with +'--sync_nodes=false' and can use REST api/kubectl cli to add/remove nodes. Node life-cycle management in the Node Controller is still under development, it is supposed to manage the Node Status Specification defined above. diff --git a/pkg/cloudprovider/controller/nodecontroller.go b/pkg/cloudprovider/controller/nodecontroller.go index 6d4cfa85539..feb64a7c608 100644 --- a/pkg/cloudprovider/controller/nodecontroller.go +++ b/pkg/cloudprovider/controller/nodecontroller.go @@ -71,14 +71,18 @@ func NewNodeController( // Run creates initial node list and start syncing instances from cloudprovider if any. // It also starts syncing cluster node status. -func (s *NodeController) Run(period time.Duration, retryCount int) { +func (s *NodeController) Run(period time.Duration, retryCount int, syncNodeList bool) { // Register intial set of nodes with their status set. var nodes *api.NodeList var err error if s.isRunningCloudProvider() { - nodes, err = s.CloudNodes() - if err != nil { - glog.Errorf("Error loading initial node from cloudprovider: %v", err) + if syncNodeList { + nodes, err = s.CloudNodes() + if err != nil { + glog.Errorf("Error loading initial node from cloudprovider: %v", err) + } + } else { + nodes = &api.NodeList{} } } else { nodes, err = s.StaticNodes() @@ -96,7 +100,7 @@ func (s *NodeController) Run(period time.Duration, retryCount int) { } // Start syncing node list from cloudprovider. - if s.isRunningCloudProvider() { + if syncNodeList && s.isRunningCloudProvider() { go util.Forever(func() { if err = s.SyncCloud(); err != nil { glog.Errorf("Error syncing cloud: %v", err) diff --git a/pkg/controllermanager/controllermanager.go b/pkg/controllermanager/controllermanager.go index 149a0027015..8b0586c4ce9 100644 --- a/pkg/controllermanager/controllermanager.go +++ b/pkg/controllermanager/controllermanager.go @@ -54,6 +54,7 @@ type CMServer struct { ResourceQuotaSyncPeriod time.Duration RegisterRetryCount int MachineList util.StringList + SyncNodeList bool // TODO: Discover these by pinging the host machines, and rip out these params. NodeMilliCPU int64 @@ -72,6 +73,7 @@ func NewCMServer() *CMServer { RegisterRetryCount: 10, NodeMilliCPU: 1000, NodeMemory: resource.MustParse("3Gi"), + SyncNodeList: true, KubeletConfig: client.KubeletConfig{ Port: ports.KubeletPort, EnableHttps: false, @@ -103,7 +105,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { client.BindClientConfigFlags(fs, &s.ClientConfig) fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") - fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs.") + fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs.") fs.DurationVar(&s.NodeSyncPeriod, "node_sync_period", s.NodeSyncPeriod, ""+ "The period for syncing nodes from cloudprovider. Longer periods will result in "+ "fewer calls to cloud provider, but may delay addition of new nodes to cluster.") @@ -111,6 +113,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.RegisterRetryCount, "register_retry_count", s.RegisterRetryCount, ""+ "The number of retries for initial node registration. Retry interval equals node_sync_period.") fs.Var(&s.MachineList, "machines", "List of machines to schedule onto, comma separated.") + fs.BoolVar(&s.SyncNodeList, "sync_nodes", s.SyncNodeList, "If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true.") // TODO: Discover these by pinging the host machines, and rip out these flags. // TODO: in the meantime, use resource.QuantityFlag() instead of these fs.Int64Var(&s.NodeMilliCPU, "node_milli_cpu", s.NodeMilliCPU, "The amount of MilliCPU provisioned on each node") @@ -119,6 +122,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { } func (s *CMServer) verifyMinionFlags() { + if !s.SyncNodeList && s.MinionRegexp != "" { + glog.Info("--minion_regexp is ignored by --sync_nodes=false") + } if s.CloudProvider == "" || s.MinionRegexp == "" { if len(s.MachineList) == 0 { glog.Info("No machines specified!") @@ -155,6 +161,7 @@ func (s *CMServer) Run(_ []string) error { if err != nil { glog.Fatalf("Failure to start kubelet client: %v", err) } + cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) nodeResources := &api.NodeResources{ Capacity: api.ResourceList{ @@ -163,7 +170,7 @@ func (s *CMServer) Run(_ []string) error { }, } nodeController := nodeControllerPkg.NewNodeController(cloud, s.MinionRegexp, s.MachineList, nodeResources, kubeClient, kubeletClient) - nodeController.Run(s.NodeSyncPeriod, s.RegisterRetryCount) + nodeController.Run(s.NodeSyncPeriod, s.RegisterRetryCount, s.SyncNodeList) resourceQuotaManager := resourcequota.NewResourceQuotaManager(kubeClient) resourceQuotaManager.Run(s.ResourceQuotaSyncPeriod)