diff --git a/pkg/registry/rbac/rest/BUILD b/pkg/registry/rbac/rest/BUILD index 3c727d46ff7..d7dcfc6e9b3 100644 --- a/pkg/registry/rbac/rest/BUILD +++ b/pkg/registry/rbac/rest/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -39,7 +40,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/component-helpers/auth/rbac/reconciliation:go_default_library", @@ -59,3 +60,13 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["storage_rbac_test.go"], + embed = [":go_default_library"], + deps = [ + "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + ], +) diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go index cfcd4b3c225..94995b9fee9 100644 --- a/pkg/registry/rbac/rest/storage_rbac.go +++ b/pkg/registry/rbac/rest/storage_rbac.go @@ -37,7 +37,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" - corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + clientset "k8s.io/client-go/kubernetes" rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/util/retry" "k8s.io/component-helpers/auth/rbac/reconciliation" @@ -174,174 +174,12 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { // initializing roles is really important. On some e2e runs, we've seen cases where etcd is down when the server // starts, the roles don't initialize, and nothing works. err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) { - failedReconciliation := false - - coreclientset, err := corev1client.NewForConfig(hookContext.LoopbackClientConfig) + client, err := clientset.NewForConfig(hookContext.LoopbackClientConfig) if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err)) + utilruntime.HandleError(fmt.Errorf("unable to initialize client set: %v", err)) return false, nil } - - clientset, err := rbacv1client.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err)) - return false, nil - } - // Make sure etcd is responding before we start reconciling - if _, err := clientset.ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) - return false, nil - } - if _, err := clientset.ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err)) - return false, nil - } - - // if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist - // in new locations - if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, clientset); err != nil { - utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err)) - return false, nil - } - - if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, clientset); err != nil { - utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err)) - return false, nil - } - - // ensure bootstrap roles are created or reconciled - for _, clusterRole := range p.ClusterRoles { - opts := reconciliation.ReconcileRoleOptions{ - Role: reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole}, - Client: reconciliation.ClusterRoleModifier{Client: clientset.ClusterRoles()}, - Confirm: true, - } - // ServiceUnavailble error is returned when the API server is blocked by storage version updates - err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { - result, err := opts.Run() - if err != nil { - return err - } - switch { - case result.Protected && result.Operation != reconciliation.ReconcileNone: - klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) - case result.Operation == reconciliation.ReconcileUpdate: - klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) - case result.Operation == reconciliation.ReconcileCreate: - klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) - } - return nil - }) - if err != nil { - // don't fail on failures, try to create as many as you can - utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err)) - failedReconciliation = true - } - } - - // ensure bootstrap rolebindings are created or reconciled - for _, clusterRoleBinding := range p.ClusterRoleBindings { - opts := reconciliation.ReconcileRoleBindingOptions{ - RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding}, - Client: reconciliation.ClusterRoleBindingClientAdapter{Client: clientset.ClusterRoleBindings()}, - Confirm: true, - } - // ServiceUnavailble error is returned when the API server is blocked by storage version updates - err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { - result, err := opts.Run() - if err != nil { - return err - } - switch { - case result.Protected && result.Operation != reconciliation.ReconcileNone: - klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) - case result.Operation == reconciliation.ReconcileUpdate: - klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) - case result.Operation == reconciliation.ReconcileCreate: - klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) - case result.Operation == reconciliation.ReconcileRecreate: - klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) - } - return nil - }) - if err != nil { - // don't fail on failures, try to create as many as you can - utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err)) - failedReconciliation = true - } - } - - // ensure bootstrap namespaced roles are created or reconciled - for namespace, roles := range p.Roles { - for _, role := range roles { - opts := reconciliation.ReconcileRoleOptions{ - Role: reconciliation.RoleRuleOwner{Role: &role}, - Client: reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()}, - Confirm: true, - } - // ServiceUnavailble error is returned when the API server is blocked by storage version updates - err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { - result, err := opts.Run() - if err != nil { - return err - } - switch { - case result.Protected && result.Operation != reconciliation.ReconcileNone: - klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) - case result.Operation == reconciliation.ReconcileUpdate: - klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) - case result.Operation == reconciliation.ReconcileCreate: - klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace) - } - return nil - }) - if err != nil { - // don't fail on failures, try to create as many as you can - utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err)) - failedReconciliation = true - } - } - } - - // ensure bootstrap namespaced rolebindings are created or reconciled - for namespace, roleBindings := range p.RoleBindings { - for _, roleBinding := range roleBindings { - opts := reconciliation.ReconcileRoleBindingOptions{ - RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding}, - Client: reconciliation.RoleBindingClientAdapter{Client: clientset, NamespaceClient: coreclientset.Namespaces()}, - Confirm: true, - } - // ServiceUnavailble error is returned when the API server is blocked by storage version updates - err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { - result, err := opts.Run() - if err != nil { - return err - } - switch { - case result.Protected && result.Operation != reconciliation.ReconcileNone: - klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) - case result.Operation == reconciliation.ReconcileUpdate: - klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) - case result.Operation == reconciliation.ReconcileCreate: - klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) - case result.Operation == reconciliation.ReconcileRecreate: - klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) - } - return nil - }) - if err != nil { - // don't fail on failures, try to create as many as you can - utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err)) - failedReconciliation = true - } - } - } - // failed to reconcile some objects, retry - if failedReconciliation { - return false, nil - } - - return true, nil + return ensureRBACPolicy(p, client) }) // if we're never able to make it through initialization, kill the API server if err != nil { @@ -352,6 +190,165 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { } } +func ensureRBACPolicy(p *PolicyData, client clientset.Interface) (done bool, err error) { + failedReconciliation := false + // Make sure etcd is responding before we start reconciling + if _, err := client.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil { + utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) + return false, nil + } + if _, err := client.RbacV1().ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil { + utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err)) + return false, nil + } + + // if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist + // in new locations + if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, client.RbacV1()); err != nil { + utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err)) + return false, nil + } + + if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, client.RbacV1()); err != nil { + utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err)) + return false, nil + } + + // ensure bootstrap roles are created or reconciled + for _, clusterRole := range p.ClusterRoles { + opts := reconciliation.ReconcileRoleOptions{ + Role: reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole}, + Client: reconciliation.ClusterRoleModifier{Client: client.RbacV1().ClusterRoles()}, + Confirm: true, + } + // ServiceUnavailble error is returned when the API server is blocked by storage version updates + err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { + result, err := opts.Run() + if err != nil { + return err + } + switch { + case result.Protected && result.Operation != reconciliation.ReconcileNone: + klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) + case result.Operation == reconciliation.ReconcileUpdate: + klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) + case result.Operation == reconciliation.ReconcileCreate: + klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) + } + return nil + }) + if err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err)) + failedReconciliation = true + } + } + + // ensure bootstrap rolebindings are created or reconciled + for _, clusterRoleBinding := range p.ClusterRoleBindings { + opts := reconciliation.ReconcileRoleBindingOptions{ + RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding}, + Client: reconciliation.ClusterRoleBindingClientAdapter{Client: client.RbacV1().ClusterRoleBindings()}, + Confirm: true, + } + // ServiceUnavailble error is returned when the API server is blocked by storage version updates + err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { + result, err := opts.Run() + if err != nil { + return err + } + switch { + case result.Protected && result.Operation != reconciliation.ReconcileNone: + klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) + case result.Operation == reconciliation.ReconcileUpdate: + klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) + case result.Operation == reconciliation.ReconcileCreate: + klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + case result.Operation == reconciliation.ReconcileRecreate: + klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + } + return nil + }) + if err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err)) + failedReconciliation = true + } + } + + // ensure bootstrap namespaced roles are created or reconciled + for namespace, roles := range p.Roles { + for _, role := range roles { + opts := reconciliation.ReconcileRoleOptions{ + Role: reconciliation.RoleRuleOwner{Role: &role}, + Client: reconciliation.RoleModifier{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()}, + Confirm: true, + } + // ServiceUnavailble error is returned when the API server is blocked by storage version updates + err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { + result, err := opts.Run() + if err != nil { + return err + } + switch { + case result.Protected && result.Operation != reconciliation.ReconcileNone: + klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) + case result.Operation == reconciliation.ReconcileUpdate: + klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) + case result.Operation == reconciliation.ReconcileCreate: + klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace) + } + return nil + }) + if err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err)) + failedReconciliation = true + } + } + } + + // ensure bootstrap namespaced rolebindings are created or reconciled + for namespace, roleBindings := range p.RoleBindings { + for _, roleBinding := range roleBindings { + opts := reconciliation.ReconcileRoleBindingOptions{ + RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding}, + Client: reconciliation.RoleBindingClientAdapter{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()}, + Confirm: true, + } + // ServiceUnavailble error is returned when the API server is blocked by storage version updates + err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error { + result, err := opts.Run() + if err != nil { + return err + } + switch { + case result.Protected && result.Operation != reconciliation.ReconcileNone: + klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) + case result.Operation == reconciliation.ReconcileUpdate: + klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) + case result.Operation == reconciliation.ReconcileCreate: + klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) + case result.Operation == reconciliation.ReconcileRecreate: + klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) + } + return nil + }) + if err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err)) + failedReconciliation = true + } + } + } + // failed to reconcile some objects, retry + if failedReconciliation { + return false, nil + } + + return true, nil +} + func (p RESTStorageProvider) GroupName() string { return rbac.GroupName } diff --git a/pkg/registry/rbac/rest/storage_rbac_test.go b/pkg/registry/rbac/rest/storage_rbac_test.go new file mode 100644 index 00000000000..11f8ce61752 --- /dev/null +++ b/pkg/registry/rbac/rest/storage_rbac_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "testing" + + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" +) + +func BenchmarkEnsureRBACPolicy(b *testing.B) { + for n := 0; n < b.N; n++ { + var policy = &PolicyData{ + ClusterRoles: append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...), + ClusterRoleBindings: append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...), + Roles: bootstrappolicy.NamespaceRoles(), + RoleBindings: bootstrappolicy.NamespaceRoleBindings(), + ClusterRolesToAggregate: bootstrappolicy.ClusterRolesToAggregate(), + ClusterRoleBindingsToSplit: bootstrappolicy.ClusterRoleBindingsToSplit(), + } + coreClientSet := fake.NewSimpleClientset() + _, _ = ensureRBACPolicy(policy, coreClientSet) + } +}