From ce499ac2861cc26b9643987faed099e82d4fc435 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Fri, 18 Oct 2019 12:59:59 +0200 Subject: [PATCH 1/2] Migrate components to EndpointsLeases leader election lock --- cmd/cloud-controller-manager/app/options/options_test.go | 2 +- cmd/kube-scheduler/app/options/options_test.go | 6 +++--- pkg/controller/apis/config/v1alpha1/defaults.go | 4 ++++ pkg/scheduler/apis/config/v1alpha1/defaults.go | 3 +++ .../src/k8s.io/component-base/config/v1alpha1/defaults.go | 1 + 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index 80eab7bafab..cb5ef9e7af6 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -49,7 +49,7 @@ func TestDefaultFlags(t *testing.T) { }, ControllerStartInterval: metav1.Duration{Duration: 0}, LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - ResourceLock: "endpoints", + ResourceLock: "endpointsleases", LeaderElect: true, LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 8e69875de63..7f5b19c2de7 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -266,7 +266,7 @@ pluginConfig: LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "endpoints", + ResourceLock: "endpointsleases", ResourceNamespace: "kube-system", ResourceName: "kube-scheduler", }, @@ -348,7 +348,7 @@ pluginConfig: LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "endpoints", + ResourceLock: "endpointsleases", ResourceNamespace: "kube-system", ResourceName: "kube-scheduler", }, @@ -411,7 +411,7 @@ pluginConfig: LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "endpoints", + ResourceLock: "endpointsleases", ResourceNamespace: "kube-system", ResourceName: "kube-scheduler", }, diff --git a/pkg/controller/apis/config/v1alpha1/defaults.go b/pkg/controller/apis/config/v1alpha1/defaults.go index 207145a341b..6074d55a009 100644 --- a/pkg/controller/apis/config/v1alpha1/defaults.go +++ b/pkg/controller/apis/config/v1alpha1/defaults.go @@ -126,6 +126,10 @@ func RecommendedDefaultGenericControllerManagerConfiguration(obj *kubectrlmgrcon obj.Controllers = []string{"*"} } + if len(obj.LeaderElection.ResourceLock) == 0 { + obj.LeaderElection.ResourceLock = "endpointsleases" + } + // Use the default ClientConnectionConfiguration and LeaderElectionConfiguration options componentbaseconfigv1alpha1.RecommendedDefaultClientConnectionConfiguration(&obj.ClientConnection) componentbaseconfigv1alpha1.RecommendedDefaultLeaderElectionConfiguration(&obj.LeaderElection) diff --git a/pkg/scheduler/apis/config/v1alpha1/defaults.go b/pkg/scheduler/apis/config/v1alpha1/defaults.go index f16a9df40b8..f7e61ce9cbb 100644 --- a/pkg/scheduler/apis/config/v1alpha1/defaults.go +++ b/pkg/scheduler/apis/config/v1alpha1/defaults.go @@ -73,6 +73,9 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubeschedulerconfigv1alpha1.Kub obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort)) } + if len(obj.LeaderElection.ResourceLock) == 0 { + obj.LeaderElection.ResourceLock = "endpointsleases" + } if len(obj.LeaderElection.LockObjectNamespace) == 0 && len(obj.LeaderElection.ResourceNamespace) == 0 { obj.LeaderElection.LockObjectNamespace = kubeschedulerconfigv1alpha1.SchedulerDefaultLockObjectNamespace } diff --git a/staging/src/k8s.io/component-base/config/v1alpha1/defaults.go b/staging/src/k8s.io/component-base/config/v1alpha1/defaults.go index caeb869a37e..258e9db4570 100644 --- a/staging/src/k8s.io/component-base/config/v1alpha1/defaults.go +++ b/staging/src/k8s.io/component-base/config/v1alpha1/defaults.go @@ -44,6 +44,7 @@ func RecommendedDefaultLeaderElectionConfiguration(obj *LeaderElectionConfigurat obj.RetryPeriod = metav1.Duration{Duration: 2 * time.Second} } if obj.ResourceLock == "" { + // TODO: Migrate to LeaseLock. obj.ResourceLock = EndpointsResourceLock } if obj.LeaderElect == nil { From fafbad45aa42e6c97a6d032ea5557ceececa9e5e Mon Sep 17 00:00:00 2001 From: wojtekt Date: Fri, 18 Oct 2019 15:46:16 +0200 Subject: [PATCH 2/2] Update bootstrappolicy RBAC rules for migration to lease API --- .../authorizer/rbac/bootstrappolicy/policy.go | 21 ++++++-- .../testdata/cluster-roles.yaml | 48 +++++++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 5509788f09d..44d39c9d9c3 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -45,6 +45,7 @@ const ( autoscalingGroup = "autoscaling" batchGroup = "batch" certificatesGroup = "certificates.k8s.io" + coordinationGroup = "coordination.k8s.io" discoveryGroup = "discovery.k8s.io" extensionsGroup = "extensions" policyGroup = "policy" @@ -172,7 +173,7 @@ func NodeRules() []rbacv1.PolicyRule { // Node leases if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) { - nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get", "create", "update", "patch", "delete").Groups("coordination.k8s.io").Resources("leases").RuleOrDie()) + nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get", "create", "update", "patch", "delete").Groups(coordinationGroup).Resources("leases").RuleOrDie()) } // RuntimeClass @@ -394,10 +395,17 @@ func ClusterRoles() []rbacv1.ClusterRole { ObjectMeta: metav1.ObjectMeta{Name: "system:kube-controller-manager"}, Rules: []rbacv1.PolicyRule{ eventsRule(), - rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints", "secrets", "serviceaccounts").RuleOrDie(), + // Needed for leader election. + rbacv1helpers.NewRule("create").Groups(coordinationGroup).Resources("leases").RuleOrDie(), + rbacv1helpers.NewRule("get", "update").Groups(coordinationGroup).Resources("leases").Names("kube-controller-manager").RuleOrDie(), + // TODO: Remove once we fully migrate to lease in leader-election. + rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), + rbacv1helpers.NewRule("get", "update").Groups(legacyGroup).Resources("endpoints").Names("kube-controller-manager").RuleOrDie(), + // Fundamental resources. + rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("secrets", "serviceaccounts").RuleOrDie(), rbacv1helpers.NewRule("delete").Groups(legacyGroup).Resources("secrets").RuleOrDie(), - rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("endpoints", "namespaces", "secrets", "serviceaccounts", "configmaps").RuleOrDie(), - rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("endpoints", "secrets", "serviceaccounts").RuleOrDie(), + rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("namespaces", "secrets", "serviceaccounts", "configmaps").RuleOrDie(), + rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("secrets", "serviceaccounts").RuleOrDie(), // Needed to check API access. These creates are non-mutating rbacv1helpers.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(), @@ -471,8 +479,11 @@ func ClusterRoles() []rbacv1.ClusterRole { eventsRule(), // This is for leaderlease access // TODO: scope this to the kube-system namespace + rbacv1helpers.NewRule("create").Groups(coordinationGroup).Resources("leases").RuleOrDie(), + rbacv1helpers.NewRule("get", "update").Groups(coordinationGroup).Resources("leases").Names("kube-scheduler").RuleOrDie(), + // TODO: Remove once we fully migrate to lease in leader-election. rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), - rbacv1helpers.NewRule("get", "update", "patch", "delete").Groups(legacyGroup).Resources("endpoints").Names("kube-scheduler").RuleOrDie(), + rbacv1helpers.NewRule("get", "update").Groups(legacyGroup).Resources("endpoints").Names("kube-scheduler").RuleOrDie(), // Fundamental resources rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("nodes").RuleOrDie(), diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index e276d15717b..6d4c4c9f356 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -510,10 +510,39 @@ items: - create - patch - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - kube-controller-manager + resources: + - leases + verbs: + - get + - update - apiGroups: - "" resources: - endpoints + verbs: + - create + - apiGroups: + - "" + resourceNames: + - kube-controller-manager + resources: + - endpoints + verbs: + - get + - update + - apiGroups: + - "" + resources: - secrets - serviceaccounts verbs: @@ -528,7 +557,6 @@ items: - "" resources: - configmaps - - endpoints - namespaces - secrets - serviceaccounts @@ -537,7 +565,6 @@ items: - apiGroups: - "" resources: - - endpoints - secrets - serviceaccounts verbs: @@ -604,6 +631,21 @@ items: - create - patch - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - kube-scheduler + resources: + - leases + verbs: + - get + - update - apiGroups: - "" resources: @@ -617,9 +659,7 @@ items: resources: - endpoints verbs: - - delete - get - - patch - update - apiGroups: - ""