github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-05 10:19:50 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #91952 from xlgao-zju/add-retries-for-updatestautes

Browse Source 
kubeadm: Add retries for kubeadm join / UpdateStatus
This commit is contained in:
Kubernetes Prow Robot 2020-06-10 22:43:34 -07:00 committed by GitHub
commit 0a5d70617f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/kubeadm/app/constants/constants.go
View File

@ -185,6 +185,10 @@ const (
TLSBootstrapTimeout = 5 * time.Minute TLSBootstrapTimeout = 5 * time.Minute
// TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check
TLSBootstrapRetryInterval = 5 * time.Second TLSBootstrapRetryInterval = 5 * time.Second
// APICallWithWriteTimeout specifies how long kubeadm should wait for api calls with at least one write
APICallWithWriteTimeout = 40 * time.Second
// APICallWithReadTimeout specifies how long kubeadm should wait for api calls with only reads
APICallWithReadTimeout = 15 * time.Second
// PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed
PullImageRetry = 5 PullImageRetry = 5

76
cmd/kubeadm/app/util/apiclient/idempotency.go
View File

@ -20,7 +20,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -62,12 +61,7 @@ func CreateOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error
// taking place) // taking place)
func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error { func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error {
var lastError error var lastError error
err := wait.ExponentialBackoff(wait.Backoff{ err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) {
Steps: 20,
Duration: 500 * time.Millisecond,
Factor: 1.0,
Jitter: 0.1,
}, func() (bool, error) {
if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil {
lastError = err lastError = err
if apierrors.IsAlreadyExists(err) { if apierrors.IsAlreadyExists(err) {
@ -89,22 +83,24 @@ func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutat
// to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was // to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was
// taking place). // taking place).
func MutateConfigMap(client clientset.Interface, meta metav1.ObjectMeta, mutator ConfigMapMutator) error { func MutateConfigMap(client clientset.Interface, meta metav1.ObjectMeta, mutator ConfigMapMutator) error {
return clientsetretry.RetryOnConflict(wait.Backoff{ var lastError error
Steps: 20, err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) {
Duration: 500 * time.Millisecond,
Factor: 1.0,
Jitter: 0.1,
}, func() error {
configMap, err := client.CoreV1().ConfigMaps(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) configMap, err := client.CoreV1().ConfigMaps(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{})
if err != nil { if err != nil {
return err lastError = err
return false, nil
} }
if err = mutator(configMap); err != nil { if err = mutator(configMap); err != nil {
return errors.Wrap(err, "unable to mutate ConfigMap") lastError = errors.Wrap(err, "unable to mutate ConfigMap")
return false, nil
} }
_, err = client.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) _, lastError = client.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
return err return lastError == nil, nil
}) })
if err == nil {
return nil
}
return lastError
} }
// CreateOrRetainConfigMap creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will retain the resource instead. // CreateOrRetainConfigMap creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will retain the resource instead.
@ -205,30 +201,48 @@ func DeleteDeploymentForeground(client clientset.Interface, namespace, name stri
// CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. // CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead.
func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error {
if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}); err != nil { var lastError error
if !apierrors.IsAlreadyExists(err) { err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) {
return errors.Wrap(err, "unable to create RBAC role") if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}); err != nil {
} if !apierrors.IsAlreadyExists(err) {
lastError = errors.Wrap(err, "unable to create RBAC role")
return false, nil
}
if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Update(context.TODO(), role, metav1.UpdateOptions{}); err != nil { if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Update(context.TODO(), role, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "unable to update RBAC role") lastError = errors.Wrap(err, "unable to update RBAC role")
return false, nil
}
} }
return true, nil
})
if err == nil {
return nil
} }
return nil return lastError
} }
// CreateOrUpdateRoleBinding creates a RoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. // CreateOrUpdateRoleBinding creates a RoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead.
func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.RoleBinding) error { func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.RoleBinding) error {
if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}); err != nil { var lastError error
if !apierrors.IsAlreadyExists(err) { err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) {
return errors.Wrap(err, "unable to create RBAC rolebinding") if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}); err != nil {
} if !apierrors.IsAlreadyExists(err) {
lastError = errors.Wrap(err, "unable to create RBAC rolebinding")
return false, nil
}
if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(context.TODO(), roleBinding, metav1.UpdateOptions{}); err != nil { if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(context.TODO(), roleBinding, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "unable to update RBAC rolebinding") lastError = errors.Wrap(err, "unable to update RBAC rolebinding")
return false, nil
}
} }
return true, nil
})
if err == nil {
return nil
} }
return nil return lastError
} }
// CreateOrUpdateClusterRole creates a ClusterRole if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. // CreateOrUpdateClusterRole creates a ClusterRole if the target resource doesn't exist. If the resource exists already, this function will update the resource instead.