Seperate loop and plugin control

Seperate loop and plugin control in the kube-controller-manager. Adding an "--external-plugin" flag to specify a plugin to load when cloud-provider is set to "external". Flag has no effect currently when the cloud-provider is not set to external. The expectation is that the cloud provider and external plugin flags would go away once all cloud providers are on stage 2 cloud-controller-manager solutions. Managing the control loops more directly based on start up flags. Addressing issue brought up by @wlan0 Switched to using the main node controller in CCM. Changes to enable full NodeController to start in CCM. Fix related tests. Unifying some common code between KCM and CCM. Fix related tests and comments. Folded in feedback from @jhorwit2 and @wlan0
2025-09-14 21:53:52 +00:00 · 2017-09-12 15:47:03 -07:00
parent eddb00e7c6
commit 070a7b5823
15 changed files with 409 additions and 284 deletions
--- a/cmd/cloud-controller-manager/app/controllermanager.go
+++ b/cmd/cloud-controller-manager/app/controllermanager.go
@@ -147,7 +147,7 @@ func Run(s *options.CloudControllerManagerServer) error {
 			clientBuilder = rootClientBuilder
 		}

-		if err := StartControllers(s, kubeconfig, clientBuilder, stop, recorder, cloud); err != nil {
+		if err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder, cloud); err != nil {
 			glog.Fatalf("error running controllers: %v", err)
 		}
 	}
@@ -193,7 +193,7 @@ func Run(s *options.CloudControllerManagerServer) error {
 }

 // StartControllers starts the cloud specific controller loops.
-func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
+func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
 	// Function to build the kube client object
 	client := func(serviceAccountName string) clientset.Interface {
 		return clientBuilder.ClientOrDie(serviceAccountName)
@@ -204,7 +204,7 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
 		cloud.Initialize(clientBuilder)
 	}

-	versionedClient := client("shared-informers")
+	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
 	sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)())

 	// Start the CloudNodeController
--- a/cmd/cloud-controller-manager/app/options/BUILD
+++ b/cmd/cloud-controller-manager/app/options/BUILD
@@ -11,7 +11,7 @@ go_library(
    srcs = ["options.go"],
    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
    deps = [
-        "//pkg/apis/componentconfig:go_default_library",
+        "//cmd/controller-manager/app/options:go_default_library",
        "//pkg/client/leaderelectionconfig:go_default_library",
        "//pkg/features:go_default_library",
        "//pkg/master/ports:go_default_library",
@@ -40,6 +40,7 @@ go_test(
    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
    library = ":go_default_library",
    deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
        "//pkg/apis/componentconfig:go_default_library",
        "//vendor/github.com/spf13/pflag:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
--- a/cmd/cloud-controller-manager/app/options/options.go
+++ b/cmd/cloud-controller-manager/app/options/options.go
@@ -21,7 +21,7 @@ import (

 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/master/ports"

@@ -33,10 +33,7 @@ import (

 // CloudControllerManagerServer is the main context object for the controller manager.
 type CloudControllerManagerServer struct {
-	componentconfig.KubeControllerManagerConfiguration
-
-	Master     string
-	Kubeconfig string
+	cmoptions.ControllerManagerServer

 	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
 	NodeStatusUpdateFrequency metav1.Duration
@@ -45,22 +42,10 @@ type CloudControllerManagerServer struct {
 // NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
 func NewCloudControllerManagerServer() *CloudControllerManagerServer {
 	s := CloudControllerManagerServer{
-		// Part of these default values also present in 'cmd/kube-controller-manager/app/options/options.go'.
-		// Please keep them in sync when doing update.
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Port:                      ports.CloudControllerManagerPort,
-			Address:                   "0.0.0.0",
-			ConcurrentServiceSyncs:    1,
-			MinResyncPeriod:           metav1.Duration{Duration: 12 * time.Hour},
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:               "kubernetes",
-			ConfigureCloudRoutes:      true,
-			ContentType:               "application/vnd.kubernetes.protobuf",
-			KubeAPIQPS:                20.0,
-			KubeAPIBurst:              30,
-			LeaderElection:            leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			ControllerStartInterval:   metav1.Duration{Duration: 0 * time.Second},
-			RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
+		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
+		// Please make common changes there and put anything cloud specific here.
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
+			KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort),
 		},
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
 	}
@@ -70,34 +55,14 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer {

 // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
 func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.")
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
+	cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
 	fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
 	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
 	fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
 	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
+
 	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)

 	utilfeature.DefaultFeatureGate.AddFlag(fs)
--- a/cmd/cloud-controller-manager/app/options/options_test.go
+++ b/cmd/cloud-controller-manager/app/options/options_test.go
@@ -25,6 +25,7 @@ import (

 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 )

@@ -65,34 +66,82 @@ func TestAddFlags(t *testing.T) {
 	f.Parse(args)

 	expected := &CloudControllerManagerServer{
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			CloudProvider:             "gce",
-			CloudConfigFile:           "/cloud-config",
-			Port:                      10000,
-			Address:                   "192.168.4.10",
-			ConcurrentServiceSyncs:    1,
-			MinResyncPeriod:           metav1.Duration{Duration: 100 * time.Minute},
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:               "k8s",
-			ConfigureCloudRoutes:      false,
-			AllocateNodeCIDRs:         true,
-			ContentType:               "application/vnd.kubernetes.protobuf",
-			EnableContentionProfiling: true,
-			KubeAPIQPS:                50.0,
-			KubeAPIBurst:              100,
-			LeaderElection: componentconfig.LeaderElectionConfiguration{
-				ResourceLock:  "configmap",
-				LeaderElect:   false,
-				LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
-				RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
-				RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
+			KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+				CloudProvider:                                   "gce",
+				CloudConfigFile:                                 "/cloud-config",
+				Port:                                            10000,
+				Address:                                         "192.168.4.10",
+				ConcurrentEndpointSyncs:                         5,
+				ConcurrentRSSyncs:                               5,
+				ConcurrentResourceQuotaSyncs:                    5,
+				ConcurrentDeploymentSyncs:                       5,
+				ConcurrentDaemonSetSyncs:                        2,
+				ConcurrentJobSyncs:                              5,
+				ConcurrentNamespaceSyncs:                        10,
+				ConcurrentSATokenSyncs:                          5,
+				ConcurrentServiceSyncs:                          1,
+				ConcurrentGCSyncs:                               20,
+				ConcurrentRCSyncs:                               5,
+				MinResyncPeriod:                                 metav1.Duration{Duration: 100 * time.Minute},
+				NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
+				ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
+				ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+				NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+				PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+				HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+				HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+				HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+				HorizontalPodAutoscalerTolerance:                0.1,
+				DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+				PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+				NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+				NodeStartupGracePeriod:                          metav1.Duration{Duration: 1 * time.Minute},
+				ClusterSigningDuration:                          metav1.Duration{Duration: 8760 * time.Hour},
+				ReconcilerSyncLoopPeriod:                        metav1.Duration{Duration: 1 * time.Minute},
+				TerminatedPodGCThreshold:                        12500,
+				RegisterRetryCount:                              10,
+				ClusterName:                                     "k8s",
+				ConfigureCloudRoutes:                            false,
+				AllocateNodeCIDRs:                               true,
+				EnableGarbageCollector:                          true,
+				EnableTaintManager:                              true,
+				HorizontalPodAutoscalerUseRESTClients:           true,
+				VolumeConfiguration: componentconfig.VolumeConfiguration{
+					EnableDynamicProvisioning:  true,
+					EnableHostPathProvisioning: false,
+					FlexVolumePluginDir:        "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+					PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+						MaximumRetry:             3,
+						MinimumTimeoutNFS:        300,
+						IncrementTimeoutNFS:      30,
+						MinimumTimeoutHostPath:   60,
+						IncrementTimeoutHostPath: 30,
+					},
+				},
+				ContentType:               "application/vnd.kubernetes.protobuf",
+				ClusterSigningCertFile:    "/etc/kubernetes/ca/ca.pem",
+				ClusterSigningKeyFile:     "/etc/kubernetes/ca/ca.key",
+				EnableContentionProfiling: true,
+				KubeAPIQPS:                50.0,
+				KubeAPIBurst:              100,
+				LeaderElection: componentconfig.LeaderElectionConfiguration{
+					ResourceLock:  "configmap",
+					LeaderElect:   false,
+					LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+					RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+					RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				},
+				ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
+				RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
+				ClusterCIDR:               "1.2.3.4/24",
+				NodeCIDRMaskSize:          24,
+				CIDRAllocatorType:         "RangeAllocator",
+				Controllers:               []string{"*"},
 			},
-			ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
-			RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
-			ClusterCIDR:               "1.2.3.4/24",
+			Kubeconfig: "/kubeconfig",
+			Master:     "192.168.4.20",
 		},
-		Kubeconfig: "/kubeconfig",
-		Master:     "192.168.4.20",
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute},
 	}
 	if !reflect.DeepEqual(expected, s) {