Merge pull request #84084 from wojtek-t/migrate_scheduler_to_endpoints_lease_lock

Migrate components to EndpointsLeases leader election lock

cmd/cloud-controller-manager/app/options/options_test.go

@@ -49,7 +49,7 @@ func TestDefaultFlags(t *testing.T) {
 				},
 				ControllerStartInterval: metav1.Duration{Duration: 0},
 				LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
-					ResourceLock:      "endpoints",
+					ResourceLock:      "endpointsleases",
 					LeaderElect:       true,
 					LeaseDuration:     metav1.Duration{Duration: 15 * time.Second},
 					RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},

cmd/kube-scheduler/app/options/options_test.go

@@ -266,7 +266,7 @@ pluginConfig:
 						LeaseDuration:     metav1.Duration{Duration: 15 * time.Second},
 						RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
 						RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
-						ResourceLock:      "endpoints",
+						ResourceLock:      "endpointsleases",
 						ResourceNamespace: "kube-system",
 						ResourceName:      "kube-scheduler",
 					},
@@ -348,7 +348,7 @@ pluginConfig:
 						LeaseDuration:     metav1.Duration{Duration: 15 * time.Second},
 						RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
 						RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
-						ResourceLock:      "endpoints",
+						ResourceLock:      "endpointsleases",
 						ResourceNamespace: "kube-system",
 						ResourceName:      "kube-scheduler",
 					},
@@ -411,7 +411,7 @@ pluginConfig:
 						LeaseDuration:     metav1.Duration{Duration: 15 * time.Second},
 						RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
 						RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
-						ResourceLock:      "endpoints",
+						ResourceLock:      "endpointsleases",
 						ResourceNamespace: "kube-system",
 						ResourceName:      "kube-scheduler",
 					},

pkg/controller/apis/config/v1alpha1/defaults.go

@@ -126,6 +126,10 @@ func RecommendedDefaultGenericControllerManagerConfiguration(obj *kubectrlmgrcon
 		obj.Controllers = []string{"*"}
 	}
 
+	if len(obj.LeaderElection.ResourceLock) == 0 {
+		obj.LeaderElection.ResourceLock = "endpointsleases"
+	}
+
 	// Use the default ClientConnectionConfiguration and LeaderElectionConfiguration options
 	componentbaseconfigv1alpha1.RecommendedDefaultClientConnectionConfiguration(&obj.ClientConnection)
 	componentbaseconfigv1alpha1.RecommendedDefaultLeaderElectionConfiguration(&obj.LeaderElection)

pkg/scheduler/apis/config/v1alpha1/defaults.go

@@ -73,6 +73,9 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubeschedulerconfigv1alpha1.Kub
 		obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort))
 	}
 
+	if len(obj.LeaderElection.ResourceLock) == 0 {
+		obj.LeaderElection.ResourceLock = "endpointsleases"
+	}
 	if len(obj.LeaderElection.LockObjectNamespace) == 0 && len(obj.LeaderElection.ResourceNamespace) == 0 {
 		obj.LeaderElection.LockObjectNamespace = kubeschedulerconfigv1alpha1.SchedulerDefaultLockObjectNamespace
 	}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -45,6 +45,7 @@ const (
 	autoscalingGroup    = "autoscaling"
 	batchGroup          = "batch"
 	certificatesGroup   = "certificates.k8s.io"
+	coordinationGroup   = "coordination.k8s.io"
 	discoveryGroup      = "discovery.k8s.io"
 	extensionsGroup     = "extensions"
 	policyGroup         = "policy"
@@ -172,7 +173,7 @@ func NodeRules() []rbacv1.PolicyRule {
 
 	// Node leases
 	if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
-		nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get", "create", "update", "patch", "delete").Groups("coordination.k8s.io").Resources("leases").RuleOrDie())
+		nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get", "create", "update", "patch", "delete").Groups(coordinationGroup).Resources("leases").RuleOrDie())
 	}
 
 	// RuntimeClass
@@ -394,10 +395,17 @@ func ClusterRoles() []rbacv1.ClusterRole {
 			ObjectMeta: metav1.ObjectMeta{Name: "system:kube-controller-manager"},
 			Rules: []rbacv1.PolicyRule{
 				eventsRule(),
-				rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints", "secrets", "serviceaccounts").RuleOrDie(),
+				// Needed for leader election.
+				rbacv1helpers.NewRule("create").Groups(coordinationGroup).Resources("leases").RuleOrDie(),
+				rbacv1helpers.NewRule("get", "update").Groups(coordinationGroup).Resources("leases").Names("kube-controller-manager").RuleOrDie(),
+				// TODO: Remove once we fully migrate to lease in leader-election.
+				rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints").RuleOrDie(),
+				rbacv1helpers.NewRule("get", "update").Groups(legacyGroup).Resources("endpoints").Names("kube-controller-manager").RuleOrDie(),
+				// Fundamental resources.
+				rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("secrets", "serviceaccounts").RuleOrDie(),
 				rbacv1helpers.NewRule("delete").Groups(legacyGroup).Resources("secrets").RuleOrDie(),
-				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("endpoints", "namespaces", "secrets", "serviceaccounts", "configmaps").RuleOrDie(),
+				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("namespaces", "secrets", "serviceaccounts", "configmaps").RuleOrDie(),
-				rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("endpoints", "secrets", "serviceaccounts").RuleOrDie(),
+				rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("secrets", "serviceaccounts").RuleOrDie(),
 				// Needed to check API access.  These creates are non-mutating
 				rbacv1helpers.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(),
 				rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(),
@@ -471,8 +479,11 @@ func ClusterRoles() []rbacv1.ClusterRole {
 		eventsRule(),
 		// This is for leaderlease access
 		// TODO: scope this to the kube-system namespace
+		rbacv1helpers.NewRule("create").Groups(coordinationGroup).Resources("leases").RuleOrDie(),
+		rbacv1helpers.NewRule("get", "update").Groups(coordinationGroup).Resources("leases").Names("kube-scheduler").RuleOrDie(),
+		// TODO: Remove once we fully migrate to lease in leader-election.
 		rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints").RuleOrDie(),
-		rbacv1helpers.NewRule("get", "update", "patch", "delete").Groups(legacyGroup).Resources("endpoints").Names("kube-scheduler").RuleOrDie(),
+		rbacv1helpers.NewRule("get", "update").Groups(legacyGroup).Resources("endpoints").Names("kube-scheduler").RuleOrDie(),
 
 		// Fundamental resources
 		rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("nodes").RuleOrDie(),

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml

@@ -510,10 +510,39 @@ items:
     - create
     - patch
     - update
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - create
+  - apiGroups:
+    - coordination.k8s.io
+    resourceNames:
+    - kube-controller-manager
+    resources:
+    - leases
+    verbs:
+    - get
+    - update
   - apiGroups:
     - ""
     resources:
     - endpoints
+    verbs:
+    - create
+  - apiGroups:
+    - ""
+    resourceNames:
+    - kube-controller-manager
+    resources:
+    - endpoints
+    verbs:
+    - get
+    - update
+  - apiGroups:
+    - ""
+    resources:
     - secrets
     - serviceaccounts
     verbs:
@@ -528,7 +557,6 @@ items:
     - ""
     resources:
     - configmaps
-    - endpoints
     - namespaces
     - secrets
     - serviceaccounts
@@ -537,7 +565,6 @@ items:
   - apiGroups:
     - ""
     resources:
-    - endpoints
     - secrets
     - serviceaccounts
     verbs:
@@ -604,6 +631,21 @@ items:
     - create
     - patch
     - update
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - create
+  - apiGroups:
+    - coordination.k8s.io
+    resourceNames:
+    - kube-scheduler
+    resources:
+    - leases
+    verbs:
+    - get
+    - update
   - apiGroups:
     - ""
     resources:
@@ -617,9 +659,7 @@ items:
     resources:
     - endpoints
     verbs:
-    - delete
     - get
-    - patch
     - update
   - apiGroups:
     - ""

staging/src/k8s.io/component-base/config/v1alpha1/defaults.go

@@ -44,6 +44,7 @@ func RecommendedDefaultLeaderElectionConfiguration(obj *LeaderElectionConfigurat
 		obj.RetryPeriod = metav1.Duration{Duration: 2 * time.Second}
 	}
 	if obj.ResourceLock == "" {
+		// TODO: Migrate to LeaseLock.
 		obj.ResourceLock = EndpointsResourceLock
 	}
 	if obj.LeaderElect == nil {