Merge pull request #51698 from rphillips/feat/lease_endpoint_reconciler

Automatic merge from submit-queue (batch tested with PRs 52240, 48145, 52220, 51698, 51777). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>..

add lease endpoint reconciler

**What this PR does / why we need it**: Adds OpenShift's LeaseEndpointReconciler to register kube-apiserver endpoints within the storage registry.

Adds a command-line argument `alpha-endpoint-reconciler-type` to the kube-apiserver.

Defaults to the old MasterCount reconciler.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes kubernetes/community#939 fixes kubernetes/kubernetes#22609

**Release note**:
```release-note
Adds a command-line argument to kube-apiserver called
--alpha-endpoint-reconciler-type=(master-count, lease, none) (default
"master-count"). The original reconciler is 'master-count'. The 'lease'
reconciler uses the storageapi and a TTL to keep alive an endpoint within the
`kube-apiserver-endpoint` storage namespace. The 'none' reconciler is a noop
reconciler that does not do anything. This is useful for self-hosted
environments.
```

/cc @lavalamp @smarterclayton @ncdc

pkg/master/master.go

@@ -50,12 +50,17 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api"
+	kapi "k8s.io/kubernetes/pkg/api"
 	coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
+	"k8s.io/kubernetes/pkg/master/reconcilers"
 	"k8s.io/kubernetes/pkg/master/tunneler"
+	"k8s.io/kubernetes/pkg/registry/core/endpoint"
+	endpointsstorage "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
 	"k8s.io/kubernetes/pkg/routes"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
 
@@ -85,6 +90,10 @@ const (
 	// DefaultEndpointReconcilerInterval is the default amount of time for how often the endpoints for
 	// the kubernetes Service are reconciled.
 	DefaultEndpointReconcilerInterval = 10 * time.Second
+	// DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer
+	DefaultEndpointReconcilerTTL = 15 * time.Second
+	// DefaultStorageEndpoint is the default storage endpoint for the lease controller
+	DefaultStorageEndpoint = "kube-apiserver-endpoint"
 )
 
 type ExtraConfig struct {
@@ -133,6 +142,19 @@ type ExtraConfig struct {
 	// Number of masters running; all masters must be started with the
 	// same value for this field. (Numbers > 1 currently untested.)
 	MasterCount int
+
+	// MasterEndpointReconcileTTL sets the time to live in seconds of an
+	// endpoint record recorded by each master. The endpoints are checked at an
+	// interval that is 2/3 of this value and this value defaults to 15s if
+	// unset. In very large clusters, this value may be increased to reduce the
+	// possibility that the master endpoint record expires (due to other load
+	// on the etcd server) and causes masters to drop in and out of the
+	// kubernetes service record. It is not recommended to set this value below
+	// 15s.
+	MasterEndpointReconcileTTL time.Duration
+
+	// Selects which reconciler to use
+	EndpointReconcilerType reconcilers.Type
 }
 
 type Config struct {
@@ -153,7 +175,7 @@ type CompletedConfig struct {
 // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be
 // used by the master.
 type EndpointReconcilerConfig struct {
-	Reconciler EndpointReconciler
+	Reconciler reconcilers.EndpointReconciler
 	Interval   time.Duration
 }
 
@@ -164,6 +186,56 @@ type Master struct {
 	ClientCARegistrationHook ClientCARegistrationHook
 }
 
+func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler {
+	endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
+	return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointClient)
+}
+
+func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler {
+	return reconcilers.NewNoneEndpointReconciler()
+}
+
+func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
+	ttl := c.ExtraConfig.MasterEndpointReconcileTTL
+	config, err := c.ExtraConfig.StorageFactory.NewConfig(kapi.Resource("apiServerIPInfo"))
+	if err != nil {
+		glog.Fatalf("Error determining service IP ranges: %v", err)
+	}
+	leaseStorage, _, err := storagefactory.Create(*config)
+	if err != nil {
+		glog.Fatalf("Error creating storage factory: %v", err)
+	}
+	endpointConfig, err := c.ExtraConfig.StorageFactory.NewConfig(kapi.Resource(DefaultStorageEndpoint))
+	if err != nil {
+		glog.Fatalf("Error getting storage config: %v", err)
+	}
+	endpointsStorage := endpointsstorage.NewREST(generic.RESTOptions{
+		StorageConfig:           endpointConfig,
+		Decorator:               generic.UndecoratedStorage,
+		DeleteCollectionWorkers: 0,
+		ResourcePrefix:          c.ExtraConfig.StorageFactory.ResourcePrefix(kapi.Resource(DefaultStorageEndpoint)),
+	})
+	endpointRegistry := endpoint.NewRegistry(endpointsStorage)
+	masterLeases := reconcilers.NewLeases(leaseStorage, "/masterleases/", ttl)
+	return reconcilers.NewLeaseEndpointReconciler(endpointRegistry, masterLeases)
+}
+
+func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
+	glog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType)
+	switch c.ExtraConfig.EndpointReconcilerType {
+	// there are numerous test dependencies that depend on a default controller
+	case "", reconcilers.MasterCountReconcilerType:
+		return c.createMasterCountReconciler()
+	case reconcilers.LeaseEndpointReconcilerType:
+		return c.createLeaseReconciler()
+	case reconcilers.NoneEndpointReconcilerType:
+		return c.createNoneReconciler()
+	default:
+		glog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType)
+	}
+	return nil
+}
+
 // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
 func (cfg *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
 	c := completedConfig{
@@ -203,10 +275,12 @@ func (cfg *Config) Complete(informers informers.SharedInformerFactory) Completed
 		c.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
 	}
 
+	if c.ExtraConfig.MasterEndpointReconcileTTL == 0 {
+		c.ExtraConfig.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL
+	}
+
 	if c.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil {
-		// use a default endpoint reconciler if nothing is set
-		endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
-		c.ExtraConfig.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointClient)
+		c.ExtraConfig.EndpointReconcilerConfig.Reconciler = cfg.createEndpointReconciler()
 	}
 
 	// this has always been hardcoded true in the past