Add benchmark for ensureRBACPolicy

This commit is contained in:
drfish 2021-01-02 10:54:40 +08:00
parent 80be1d6c72
commit 5711625518
3 changed files with 214 additions and 167 deletions

View File

@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
@ -39,7 +40,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library",
"//staging/src/k8s.io/client-go/util/retry:go_default_library",
"//staging/src/k8s.io/component-helpers/auth/rbac/reconciliation:go_default_library",
@ -59,3 +60,13 @@ filegroup(
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["storage_rbac_test.go"],
embed = [":go_default_library"],
deps = [
"//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

View File

@ -37,7 +37,7 @@ import (
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
serverstorage "k8s.io/apiserver/pkg/server/storage"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
clientset "k8s.io/client-go/kubernetes"
rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1"
"k8s.io/client-go/util/retry"
"k8s.io/component-helpers/auth/rbac/reconciliation"
@ -174,174 +174,12 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
// initializing roles is really important. On some e2e runs, we've seen cases where etcd is down when the server
// starts, the roles don't initialize, and nothing works.
err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) {
failedReconciliation := false
coreclientset, err := corev1client.NewForConfig(hookContext.LoopbackClientConfig)
client, err := clientset.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err))
utilruntime.HandleError(fmt.Errorf("unable to initialize client set: %v", err))
return false, nil
}
clientset, err := rbacv1client.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err))
return false, nil
}
// Make sure etcd is responding before we start reconciling
if _, err := clientset.ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err))
return false, nil
}
if _, err := clientset.ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err))
return false, nil
}
// if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist
// in new locations
if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, clientset); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err))
return false, nil
}
if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, clientset); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err))
return false, nil
}
// ensure bootstrap roles are created or reconciled
for _, clusterRole := range p.ClusterRoles {
opts := reconciliation.ReconcileRoleOptions{
Role: reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole},
Client: reconciliation.ClusterRoleModifier{Client: clientset.ClusterRoles()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err))
failedReconciliation = true
}
}
// ensure bootstrap rolebindings are created or reconciled
for _, clusterRoleBinding := range p.ClusterRoleBindings {
opts := reconciliation.ReconcileRoleBindingOptions{
RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding},
Client: reconciliation.ClusterRoleBindingClientAdapter{Client: clientset.ClusterRoleBindings()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
case result.Operation == reconciliation.ReconcileRecreate:
klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err))
failedReconciliation = true
}
}
// ensure bootstrap namespaced roles are created or reconciled
for namespace, roles := range p.Roles {
for _, role := range roles {
opts := reconciliation.ReconcileRoleOptions{
Role: reconciliation.RoleRuleOwner{Role: &role},
Client: reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
failedReconciliation = true
}
}
}
// ensure bootstrap namespaced rolebindings are created or reconciled
for namespace, roleBindings := range p.RoleBindings {
for _, roleBinding := range roleBindings {
opts := reconciliation.ReconcileRoleBindingOptions{
RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding},
Client: reconciliation.RoleBindingClientAdapter{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
case result.Operation == reconciliation.ReconcileRecreate:
klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err))
failedReconciliation = true
}
}
}
// failed to reconcile some objects, retry
if failedReconciliation {
return false, nil
}
return true, nil
return ensureRBACPolicy(p, client)
})
// if we're never able to make it through initialization, kill the API server
if err != nil {
@ -352,6 +190,165 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
}
}
func ensureRBACPolicy(p *PolicyData, client clientset.Interface) (done bool, err error) {
failedReconciliation := false
// Make sure etcd is responding before we start reconciling
if _, err := client.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err))
return false, nil
}
if _, err := client.RbacV1().ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err))
return false, nil
}
// if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist
// in new locations
if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, client.RbacV1()); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err))
return false, nil
}
if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, client.RbacV1()); err != nil {
utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err))
return false, nil
}
// ensure bootstrap roles are created or reconciled
for _, clusterRole := range p.ClusterRoles {
opts := reconciliation.ReconcileRoleOptions{
Role: reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole},
Client: reconciliation.ClusterRoleModifier{Client: client.RbacV1().ClusterRoles()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err))
failedReconciliation = true
}
}
// ensure bootstrap rolebindings are created or reconciled
for _, clusterRoleBinding := range p.ClusterRoleBindings {
opts := reconciliation.ReconcileRoleBindingOptions{
RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding},
Client: reconciliation.ClusterRoleBindingClientAdapter{Client: client.RbacV1().ClusterRoleBindings()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
case result.Operation == reconciliation.ReconcileRecreate:
klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err))
failedReconciliation = true
}
}
// ensure bootstrap namespaced roles are created or reconciled
for namespace, roles := range p.Roles {
for _, role := range roles {
opts := reconciliation.ReconcileRoleOptions{
Role: reconciliation.RoleRuleOwner{Role: &role},
Client: reconciliation.RoleModifier{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
failedReconciliation = true
}
}
}
// ensure bootstrap namespaced rolebindings are created or reconciled
for namespace, roleBindings := range p.RoleBindings {
for _, roleBinding := range roleBindings {
opts := reconciliation.ReconcileRoleBindingOptions{
RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding},
Client: reconciliation.RoleBindingClientAdapter{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
Confirm: true,
}
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run()
if err != nil {
return err
}
switch {
case result.Protected && result.Operation != reconciliation.ReconcileNone:
klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileUpdate:
klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
case result.Operation == reconciliation.ReconcileCreate:
klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
case result.Operation == reconciliation.ReconcileRecreate:
klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
}
return nil
})
if err != nil {
// don't fail on failures, try to create as many as you can
utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err))
failedReconciliation = true
}
}
}
// failed to reconcile some objects, retry
if failedReconciliation {
return false, nil
}
return true, nil
}
func (p RESTStorageProvider) GroupName() string {
return rbac.GroupName
}

View File

@ -0,0 +1,39 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rest
import (
"testing"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy"
)
func BenchmarkEnsureRBACPolicy(b *testing.B) {
for n := 0; n < b.N; n++ {
var policy = &PolicyData{
ClusterRoles: append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...),
ClusterRoleBindings: append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...),
Roles: bootstrappolicy.NamespaceRoles(),
RoleBindings: bootstrappolicy.NamespaceRoleBindings(),
ClusterRolesToAggregate: bootstrappolicy.ClusterRolesToAggregate(),
ClusterRoleBindingsToSplit: bootstrappolicy.ClusterRoleBindingsToSplit(),
}
coreClientSet := fake.NewSimpleClientset()
_, _ = ensureRBACPolicy(policy, coreClientSet)
}
}