github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 18:31:15 +00:00
Code Issues Projects Releases Wiki Activity

make some rbac and scheduling post start hooks tolerate the apiserver

Browse Source 
bootstrap delay caused by installing storage versions.
This commit is contained in:
Chao Xu 2020-02-17 14:19:32 -08:00 committed by Haowei Cai
parent 7218978716
commit 22452917c2
2 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/registry/rbac/rest/storage_rbac.go
View File

@ -26,6 +26,7 @@ import (
rbacapiv1 "k8s.io/api/rbac/v1" rbacapiv1 "k8s.io/api/rbac/v1"
rbacapiv1alpha1 "k8s.io/api/rbac/v1alpha1" rbacapiv1alpha1 "k8s.io/api/rbac/v1alpha1"
rbacapiv1beta1 "k8s.io/api/rbac/v1beta1" rbacapiv1beta1 "k8s.io/api/rbac/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
@ -160,6 +161,14 @@ type PolicyData struct {
ClusterRoleBindingsToSplit map[string]rbacapiv1.ClusterRoleBinding ClusterRoleBindingsToSplit map[string]rbacapiv1.ClusterRoleBinding
} }
func isConflictOrServiceUnavailable(err error) bool {
return errors.IsConflict(err) || errors.IsServiceUnavailable(err)
}
func retryOnConflictOrServiceUnavailable(backoff wait.Backoff, fn func() error) error {
return retry.OnError(backoff, isConflictOrServiceUnavailable, fn)
}
func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
return func(hookContext genericapiserver.PostStartHookContext) error { return func(hookContext genericapiserver.PostStartHookContext) error {
// initializing roles is really important. On some e2e runs, we've seen cases where etcd is down when the server // initializing roles is really important. On some e2e runs, we've seen cases where etcd is down when the server
@ -206,7 +215,8 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
Client: reconciliation.ClusterRoleModifier{Client: clientset.ClusterRoles()}, Client: reconciliation.ClusterRoleModifier{Client: clientset.ClusterRoles()},
Confirm: true, Confirm: true,
} }
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run() result, err := opts.Run()
if err != nil { if err != nil {
return err return err
@ -234,7 +244,8 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
Client: reconciliation.ClusterRoleBindingClientAdapter{Client: clientset.ClusterRoleBindings()}, Client: reconciliation.ClusterRoleBindingClientAdapter{Client: clientset.ClusterRoleBindings()},
Confirm: true, Confirm: true,
} }
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run() result, err := opts.Run()
if err != nil { if err != nil {
return err return err
@ -265,7 +276,8 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
Client: reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()}, Client: reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
Confirm: true, Confirm: true,
} }
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run() result, err := opts.Run()
if err != nil { if err != nil {
return err return err
@ -295,7 +307,8 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
Client: reconciliation.RoleBindingClientAdapter{Client: clientset, NamespaceClient: coreclientset.Namespaces()}, Client: reconciliation.RoleBindingClientAdapter{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
Confirm: true, Confirm: true,
} }
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // ServiceUnavailble error is returned when the API server is blocked by storage version updates
err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
result, err := opts.Run() result, err := opts.Run()
if err != nil { if err != nil {
return err return err

11
pkg/registry/scheduling/rest/storage_scheduling.go
View File

@ -128,11 +128,16 @@ func AddSystemPriorityClasses() genericapiserver.PostStartHookFunc {
if err != nil { if err != nil {
if apierrors.IsNotFound(err) { if apierrors.IsNotFound(err) {
_, err := schedClientSet.PriorityClasses().Create(context.TODO(), pc, metav1.CreateOptions{}) _, err := schedClientSet.PriorityClasses().Create(context.TODO(), pc, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) { if err == nil || apierrors.IsAlreadyExists(err) {
return false, err
} else {
klog.Infof("created PriorityClass %s with value %v", pc.Name, pc.Value) klog.Infof("created PriorityClass %s with value %v", pc.Name, pc.Value)
continue
} }
// ServiceUnavailble error is returned when the API server is blocked by storage version updates
if apierrors.IsServiceUnavailable(err) {
klog.Infof("going to retry, unable to create PriorityClass %s: %v", pc.Name, err)
return false, nil
}
return false, err
} else { } else {
// Unable to get the priority class for reasons other than "not found". // Unable to get the priority class for reasons other than "not found".
klog.Warningf("unable to get PriorityClass %v: %v. Retrying...", pc.Name, err) klog.Warningf("unable to get PriorityClass %v: %v. Retrying...", pc.Name, err)