Merge pull request #3498 from pravisankar/ravi/sync-nodes-optional

Make periodic sync nodes from -cloud_provider/-machines optional.

cluster/saltbase/salt/kube-controller-manager/default

@@ -9,6 +9,7 @@
 {% set machines = ""-%}
 {% set cloud_provider = "" -%}
 {% set minion_regexp = "--minion_regexp=.*" -%}
+{% set sync_nodes = "--sync_nodes=true" -%}
 {% if grains.cloud_provider is defined -%}
   {% set cloud_provider = "--cloud_provider=" + grains.cloud_provider -%}
 {% endif -%}
@@ -47,4 +48,4 @@
 {% endif -%}
 {% endif -%}
 
-DAEMON_ARGS="{{daemon_args}} {{master}} {{machines}} {{ minion_regexp }} {{ cloud_provider }} {{ cloud_config }} {{pillar['log_level']}}"
+DAEMON_ARGS="{{daemon_args}} {{master}} {{machines}} {{ minion_regexp }} {{ cloud_provider }} {{ sync_nodes }} {{ cloud_config }} {{pillar['log_level']}}"

cmd/integration/integration.go

@@ -203,7 +203,7 @@ func startComponents(manifestURL string) (apiServerURL string) {
 
 	nodeResources := &api.NodeResources{}
 	nodeController := nodeControllerPkg.NewNodeController(nil, "", machineList, nodeResources, cl, fakeKubeletClient{})
-	nodeController.Run(5*time.Second, 10)
+	nodeController.Run(5*time.Second, 10, true)
 
 	// Kubelet (localhost)
 	testRootDir := makeTempDirOrDie("kubelet_integ_1.")

cmd/kubernetes/kubernetes.go

@@ -124,7 +124,7 @@ func runControllerManager(machineList []string, cl *client.Client, nodeMilliCPU,
 	}
 	kubeClient := &client.HTTPKubeletClient{Client: http.DefaultClient, Port: ports.KubeletPort}
 	nodeController := nodeControllerPkg.NewNodeController(nil, "", machineList, nodeResources, cl, kubeClient)
-	nodeController.Run(10*time.Second, 10)
+	nodeController.Run(10*time.Second, 10, true)
 
 	endpoints := service.NewEndpointController(cl)
 	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

docs/man/kube-controller-manager.1.md

@@ -36,7 +36,7 @@ The kube-controller-manager has several options.
 	The provider for cloud services. Empty string for no provider.
 
 **--minion_regexp**=""
-	If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs.
+	If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs.
 
 **--insecure_skip_tls_verify**=false
 	If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
@@ -56,6 +56,9 @@ The kube-controller-manager has several options.
 **--machines**=[]
     List of machines to schedule onto, comma separated.
 
+**--sync_nodes**=true
+        If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true.
+
 **--master**=""
 	The address of the Kubernetes API server.
 

docs/node.md

@@ -110,9 +110,11 @@ any binary; therefore, to
 join Kubernetes cluster, you as an admin need to make sure proper services are
 running in the node. In the future, we plan to automatically provision some node
 services. In case of no cloud provider, Node Controller simply registers all
-machines from `-machines` flag, any futher interactions need to be done manually
+machines from `--machines` flag, any futher interactions need to be done manually
-by using `kubectl`. If you are paranoid, leave `-machines` empty and create all
+by using `kubectl`. If you are paranoid, leave `--machines` empty and create all
 machines from `kubectl` one by one - the two approaches are equivalent.
+Optionally you can skip cluster-wide node synchronization with
+'--sync_nodes=false' and can use REST api/kubectl cli to add/remove nodes.
 
 Node life-cycle management in the Node Controller is still under development, it
 is supposed to manage the Node Status Specification defined above.

pkg/cloudprovider/controller/nodecontroller.go

@@ -71,14 +71,18 @@ func NewNodeController(
 
 // Run creates initial node list and start syncing instances from cloudprovider if any.
 // It also starts syncing cluster node status.
-func (s *NodeController) Run(period time.Duration, retryCount int) {
+func (s *NodeController) Run(period time.Duration, retryCount int, syncNodeList bool) {
 	// Register intial set of nodes with their status set.
 	var nodes *api.NodeList
 	var err error
 	if s.isRunningCloudProvider() {
-		nodes, err = s.CloudNodes()
+		if syncNodeList {
-		if err != nil {
+			nodes, err = s.CloudNodes()
-			glog.Errorf("Error loading initial node from cloudprovider: %v", err)
+			if err != nil {
+				glog.Errorf("Error loading initial node from cloudprovider: %v", err)
+			}
+		} else {
+			nodes = &api.NodeList{}
 		}
 	} else {
 		nodes, err = s.StaticNodes()
@@ -96,7 +100,7 @@ func (s *NodeController) Run(period time.Duration, retryCount int) {
 	}
 
 	// Start syncing node list from cloudprovider.
-	if s.isRunningCloudProvider() {
+	if syncNodeList && s.isRunningCloudProvider() {
 		go util.Forever(func() {
 			if err = s.SyncCloud(); err != nil {
 				glog.Errorf("Error syncing cloud: %v", err)

pkg/controllermanager/controllermanager.go

@@ -54,6 +54,7 @@ type CMServer struct {
 	ResourceQuotaSyncPeriod time.Duration
 	RegisterRetryCount      int
 	MachineList             util.StringList
+	SyncNodeList            bool
 
 	// TODO: Discover these by pinging the host machines, and rip out these params.
 	NodeMilliCPU int64
@@ -72,6 +73,7 @@ func NewCMServer() *CMServer {
 		RegisterRetryCount:      10,
 		NodeMilliCPU:            1000,
 		NodeMemory:              resource.MustParse("3Gi"),
+		SyncNodeList:            true,
 		KubeletConfig: client.KubeletConfig{
 			Port:        ports.KubeletPort,
 			EnableHttps: false,
@@ -103,7 +105,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
 	client.BindClientConfigFlags(fs, &s.ClientConfig)
 	fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
 	fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
-	fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and -cloud_provider is specified, a regular expression for matching minion VMs.")
+	fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs.")
 	fs.DurationVar(&s.NodeSyncPeriod, "node_sync_period", s.NodeSyncPeriod, ""+
 		"The period for syncing nodes from cloudprovider. Longer periods will result in "+
 		"fewer calls to cloud provider, but may delay addition of new nodes to cluster.")
@@ -111,6 +113,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.RegisterRetryCount, "register_retry_count", s.RegisterRetryCount, ""+
 		"The number of retries for initial node registration.  Retry interval equals node_sync_period.")
 	fs.Var(&s.MachineList, "machines", "List of machines to schedule onto, comma separated.")
+	fs.BoolVar(&s.SyncNodeList, "sync_nodes", s.SyncNodeList, "If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true.")
 	// TODO: Discover these by pinging the host machines, and rip out these flags.
 	// TODO: in the meantime, use resource.QuantityFlag() instead of these
 	fs.Int64Var(&s.NodeMilliCPU, "node_milli_cpu", s.NodeMilliCPU, "The amount of MilliCPU provisioned on each node")
@@ -119,6 +122,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
 }
 
 func (s *CMServer) verifyMinionFlags() {
+	if !s.SyncNodeList && s.MinionRegexp != "" {
+		glog.Info("--minion_regexp is ignored by --sync_nodes=false")
+	}
 	if s.CloudProvider == "" || s.MinionRegexp == "" {
 		if len(s.MachineList) == 0 {
 			glog.Info("No machines specified!")
@@ -155,6 +161,7 @@ func (s *CMServer) Run(_ []string) error {
 	if err != nil {
 		glog.Fatalf("Failure to start kubelet client: %v", err)
 	}
+
 	cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
 	nodeResources := &api.NodeResources{
 		Capacity: api.ResourceList{
@@ -163,7 +170,7 @@ func (s *CMServer) Run(_ []string) error {
 		},
 	}
 	nodeController := nodeControllerPkg.NewNodeController(cloud, s.MinionRegexp, s.MachineList, nodeResources, kubeClient, kubeletClient)
-	nodeController.Run(s.NodeSyncPeriod, s.RegisterRetryCount)
+	nodeController.Run(s.NodeSyncPeriod, s.RegisterRetryCount, s.SyncNodeList)
 
 	resourceQuotaManager := resourcequota.NewResourceQuotaManager(kubeClient)
 	resourceQuotaManager.Run(s.ResourceQuotaSyncPeriod)