diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 984b6680590..9cbfa44a3ed 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -260,9 +260,10 @@ func TestAddFlags(t *testing.T) { ClientCA: "/client-ca", }, WebHook: &kubeoptions.WebHookAuthenticationOptions{ - CacheTTL: 180000000000, - ConfigFile: "/token-webhook-config", - Version: "v1beta1", + CacheTTL: 180000000000, + ConfigFile: "/token-webhook-config", + Version: "v1beta1", + RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), }, BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, OIDC: &kubeoptions.OIDCAuthenticationOptions{ @@ -284,6 +285,7 @@ func TestAddFlags(t *testing.T) { WebhookCacheAuthorizedTTL: 180000000000, WebhookCacheUnauthorizedTTL: 60000000000, WebhookVersion: "v1beta1", + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), }, CloudProvider: &kubeoptions.CloudProviderOptions{ CloudConfigFile: "/cloud-config", diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 4e32ba5b79e..871a2ae1e2d 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -405,8 +405,9 @@ func TestAddFlags(t *testing.T) { BindNetwork: "tcp", }).WithLoopback(), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + CacheTTL: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ UsernameHeaders: []string{"x-remote-user"}, GroupHeaders: []string{"x-remote-group"}, @@ -418,6 +419,7 @@ func TestAddFlags(t *testing.T) { AllowCacheTTL: 10 * time.Second, DenyCacheTTL: 10 * time.Second, ClientTimeout: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), RemoteKubeConfigFileOptional: true, AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* }, diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 0e9526f117c..569dd2d4f02 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -95,6 +95,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library", diff --git a/cmd/kubelet/app/auth.go b/cmd/kubelet/app/auth.go index 6eadf29bb1b..76b35f5db98 100644 --- a/cmd/kubelet/app/auth.go +++ b/cmd/kubelet/app/auth.go @@ -27,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" "k8s.io/apiserver/pkg/server/dynamiccertificates" + genericoptions "k8s.io/apiserver/pkg/server/options" clientset "k8s.io/client-go/kubernetes" authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1" authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1" @@ -84,6 +85,7 @@ func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletc if client == nil { return nil, nil, errors.New("no client provided, cannot use webhook authentication") } + authenticatorConfig.WebhookRetryBackoff = genericoptions.DefaultAuthWebhookRetryBackoff() authenticatorConfig.TokenAccessReviewClient = client } @@ -113,6 +115,7 @@ func BuildAuthz(client authorizationclient.SubjectAccessReviewInterface, authz k SubjectAccessReviewClient: client, AllowCacheTTL: authz.Webhook.CacheAuthorizedTTL.Duration, DenyCacheTTL: authz.Webhook.CacheUnauthorizedTTL.Duration, + WebhookRetryBackoff: genericoptions.DefaultAuthWebhookRetryBackoff(), } return authorizerConfig.New() diff --git a/pkg/kubeapiserver/authenticator/BUILD b/pkg/kubeapiserver/authenticator/BUILD index 4d6525b6a05..f99c3bb755c 100644 --- a/pkg/kubeapiserver/authenticator/BUILD +++ b/pkg/kubeapiserver/authenticator/BUILD @@ -12,6 +12,7 @@ go_library( deps = [ "//pkg/serviceaccount:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/group:go_default_library", diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go index 60c710f7972..bc8033c04ca 100644 --- a/pkg/kubeapiserver/authenticator/config.go +++ b/pkg/kubeapiserver/authenticator/config.go @@ -17,11 +17,13 @@ limitations under the License. package authenticator import ( + "errors" "time" "github.com/go-openapi/spec" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/authenticatorfactory" "k8s.io/apiserver/pkg/authentication/group" @@ -66,6 +68,10 @@ type Config struct { WebhookTokenAuthnConfigFile string WebhookTokenAuthnVersion string WebhookTokenAuthnCacheTTL time.Duration + // WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff TokenSuccessCacheTTL time.Duration TokenFailureCacheTTL time.Duration @@ -280,7 +286,11 @@ func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences } func newWebhookTokenAuthenticator(config Config) (authenticator.Token, error) { - webhookTokenAuthenticator, err := webhook.New(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnVersion, config.APIAudiences, config.CustomDial) + if config.WebhookRetryBackoff == nil { + return nil, errors.New("retry backoff parameters for authentication webhook has not been specified") + } + + webhookTokenAuthenticator, err := webhook.New(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnVersion, config.APIAudiences, *config.WebhookRetryBackoff, config.CustomDial) if err != nil { return nil, err } diff --git a/pkg/kubeapiserver/authorizer/BUILD b/pkg/kubeapiserver/authorizer/BUILD index 06e5735e1a1..13c2eb80e4d 100644 --- a/pkg/kubeapiserver/authorizer/BUILD +++ b/pkg/kubeapiserver/authorizer/BUILD @@ -17,6 +17,7 @@ go_library( "//plugin/pkg/auth/authorizer/rbac:go_default_library", "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/union:go_default_library", diff --git a/pkg/kubeapiserver/authorizer/config.go b/pkg/kubeapiserver/authorizer/config.go index 0a1b1ca85c2..17aca6edef3 100644 --- a/pkg/kubeapiserver/authorizer/config.go +++ b/pkg/kubeapiserver/authorizer/config.go @@ -17,10 +17,12 @@ limitations under the License. package authorizer import ( + "errors" "fmt" "time" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" "k8s.io/apiserver/pkg/authorization/union" @@ -53,6 +55,10 @@ type Config struct { WebhookCacheAuthorizedTTL time.Duration // TTL for caching of unauthorized responses from the webhook server. WebhookCacheUnauthorizedTTL time.Duration + // WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff VersionedInformerFactory versionedinformers.SharedInformerFactory @@ -104,10 +110,14 @@ func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, erro authorizers = append(authorizers, abacAuthorizer) ruleResolvers = append(ruleResolvers, abacAuthorizer) case modes.ModeWebhook: + if config.WebhookRetryBackoff == nil { + return nil, nil, errors.New("retry backoff parameters for authorization webhook has not been specified") + } webhookAuthorizer, err := webhook.New(config.WebhookConfigFile, config.WebhookVersion, config.WebhookCacheAuthorizedTTL, config.WebhookCacheUnauthorizedTTL, + *config.WebhookRetryBackoff, config.CustomDial) if err != nil { return nil, nil, err diff --git a/pkg/kubeapiserver/options/BUILD b/pkg/kubeapiserver/options/BUILD index eee2511fcf4..5c9e9ba6fd8 100644 --- a/pkg/kubeapiserver/options/BUILD +++ b/pkg/kubeapiserver/options/BUILD @@ -53,6 +53,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/resourcequota:go_default_library", diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index fc368cc3d19..d0ec47b5f3b 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/egressselector" @@ -104,6 +105,11 @@ type WebHookAuthenticationOptions struct { ConfigFile string Version string CacheTTL time.Duration + + // RetryBackoff specifies the backoff parameters for the authentication webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + RetryBackoff *wait.Backoff } // NewBuiltInAuthenticationOptions create a new BuiltInAuthenticationOptions, just set default token cache TTL @@ -172,8 +178,9 @@ func (o *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOpt // WithWebHook set default value for web hook authentication func (o *BuiltInAuthenticationOptions) WithWebHook() *BuiltInAuthenticationOptions { o.WebHook = &WebHookAuthenticationOptions{ - Version: "v1beta1", - CacheTTL: 2 * time.Minute, + Version: "v1beta1", + CacheTTL: 2 * time.Minute, + RetryBackoff: genericoptions.DefaultAuthWebhookRetryBackoff(), } return o } @@ -216,6 +223,13 @@ func (o *BuiltInAuthenticationOptions) Validate() []error { } } + if o.WebHook != nil { + retryBackoff := o.WebHook.RetryBackoff + if retryBackoff != nil && retryBackoff.Steps <= 0 { + allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", retryBackoff.Steps)) + } + } + return allErrors } @@ -415,6 +429,7 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat ret.WebhookTokenAuthnConfigFile = o.WebHook.ConfigFile ret.WebhookTokenAuthnVersion = o.WebHook.Version ret.WebhookTokenAuthnCacheTTL = o.WebHook.CacheTTL + ret.WebhookRetryBackoff = o.WebHook.RetryBackoff if len(o.WebHook.ConfigFile) > 0 && o.WebHook.CacheTTL > 0 { if o.TokenSuccessCacheTTL > 0 && o.WebHook.CacheTTL < o.TokenSuccessCacheTTL { diff --git a/pkg/kubeapiserver/options/authorization.go b/pkg/kubeapiserver/options/authorization.go index b21690464f2..e9d4f39bc53 100644 --- a/pkg/kubeapiserver/options/authorization.go +++ b/pkg/kubeapiserver/options/authorization.go @@ -24,6 +24,8 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + genericoptions "k8s.io/apiserver/pkg/server/options" versionedinformers "k8s.io/client-go/informers" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" @@ -37,6 +39,10 @@ type BuiltInAuthorizationOptions struct { WebhookVersion string WebhookCacheAuthorizedTTL time.Duration WebhookCacheUnauthorizedTTL time.Duration + // WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff } // NewBuiltInAuthorizationOptions create a BuiltInAuthorizationOptions with default value @@ -46,6 +52,7 @@ func NewBuiltInAuthorizationOptions() *BuiltInAuthorizationOptions { WebhookVersion: "v1beta1", WebhookCacheAuthorizedTTL: 5 * time.Minute, WebhookCacheUnauthorizedTTL: 30 * time.Second, + WebhookRetryBackoff: genericoptions.DefaultAuthWebhookRetryBackoff(), } } @@ -89,6 +96,10 @@ func (o *BuiltInAuthorizationOptions) Validate() []error { allErrors = append(allErrors, fmt.Errorf("authorization-mode %q has mode specified more than once", o.Modes)) } + if o.WebhookRetryBackoff != nil && o.WebhookRetryBackoff.Steps <= 0 { + allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", o.WebhookRetryBackoff.Steps)) + } + return allErrors } @@ -127,5 +138,6 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac WebhookCacheAuthorizedTTL: o.WebhookCacheAuthorizedTTL, WebhookCacheUnauthorizedTTL: o.WebhookCacheUnauthorizedTTL, VersionedInformerFactory: versionedInformerFactory, + WebhookRetryBackoff: o.WebhookRetryBackoff, } } diff --git a/plugin/pkg/admission/imagepolicy/admission.go b/plugin/pkg/admission/imagepolicy/admission.go index 572fe67e77e..80bd4852396 100644 --- a/plugin/pkg/admission/imagepolicy/admission.go +++ b/plugin/pkg/admission/imagepolicy/admission.go @@ -261,7 +261,8 @@ func NewImagePolicyWebhook(configFile io.Reader) (*Plugin, error) { return nil, err } - gw, err := webhook.NewGenericWebhook(legacyscheme.Scheme, legacyscheme.Codecs, whConfig.KubeConfigFile, groupVersions, whConfig.RetryBackoff, nil) + retryBackoff := webhook.DefaultRetryBackoffWithInitialDelay(whConfig.RetryBackoff) + gw, err := webhook.NewGenericWebhook(legacyscheme.Scheme, legacyscheme.Codecs, whConfig.KubeConfigFile, groupVersions, retryBackoff, nil) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD index 891ce7e279e..e5ea7a93c44 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD @@ -15,6 +15,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/authentication/authenticatorfactory", importpath = "k8s.io/apiserver/pkg/authentication/authenticatorfactory", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/group:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/anonymous:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go index b9c7e2e6eee..83697bb5470 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go @@ -22,6 +22,7 @@ import ( "github.com/go-openapi/spec" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/group" "k8s.io/apiserver/pkg/authentication/request/anonymous" @@ -43,6 +44,11 @@ type DelegatingAuthenticatorConfig struct { // TokenAccessReviewClient is a client to do token review. It can be nil. Then every token is ignored. TokenAccessReviewClient authenticationclient.TokenReviewInterface + // WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff + // CacheTTL is the length of time that a token authentication answer will be cached. CacheTTL time.Duration @@ -79,7 +85,10 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur } if c.TokenAccessReviewClient != nil { - tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences) + if c.WebhookRetryBackoff == nil { + return nil, nil, errors.New("retry backoff parameters for delegating authentication webhook has not been specified") + } + tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences, *c.WebhookRetryBackoff) if err != nil { return nil, nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/BUILD b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/BUILD index 06b8f73a5f6..2d1f1e6c9a2 100644 --- a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/BUILD @@ -25,6 +25,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory", importpath = "k8s.io/apiserver/pkg/authorization/authorizerfactory", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go index fa385e12554..665483308f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go +++ b/staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go @@ -17,8 +17,10 @@ limitations under the License. package authorizerfactory import ( + "errors" "time" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/plugin/pkg/authorizer/webhook" authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1" @@ -35,12 +37,22 @@ type DelegatingAuthorizerConfig struct { // DenyCacheTTL is the length of time that an unsuccessful authorization response will be cached. // You generally want more responsive, "deny, try again" flows. DenyCacheTTL time.Duration + + // WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff } func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) { + if c.WebhookRetryBackoff == nil { + return nil, errors.New("retry backoff parameters for delegating authorization webhook has not been specified") + } + return webhook.NewFromInterface( c.SubjectAccessReviewClient, c.AllowCacheTTL, c.DenyCacheTTL, + *c.WebhookRetryBackoff, ) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 84a507fe444..bd4fcd1ff72 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -34,6 +34,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/metrics:go_default_library", @@ -69,6 +70,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/log:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go index 126bc3ed44a..c3a709dcf10 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go @@ -37,6 +37,7 @@ import ( "k8s.io/apiserver/pkg/audit/policy" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/egressselector" + "k8s.io/apiserver/pkg/util/webhook" pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" pluginlog "k8s.io/apiserver/plugin/pkg/audit/log" plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" @@ -154,7 +155,7 @@ type AuditDynamicOptions struct { func NewAuditOptions() *AuditOptions { return &AuditOptions{ WebhookOptions: AuditWebhookOptions{ - InitialBackoff: pluginwebhook.DefaultInitialBackoff, + InitialBackoff: pluginwebhook.DefaultInitialBackoffDelay, BatchOptions: AuditBatchOptions{ Mode: ModeBatch, BatchConfig: defaultWebhookBatchConfig(), @@ -569,7 +570,7 @@ func (o *AuditWebhookOptions) enabled() bool { // this is done so that the same trucate backend can wrap both the webhook and dynamic backends func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc) (audit.Backend, error) { groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString) - webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, o.InitialBackoff, customDial) + webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, webhook.DefaultRetryBackoffWithInitialDelay(o.InitialBackoff), customDial) if err != nil { return nil, fmt.Errorf("initializing audit webhook: %v", err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go index 1929e975058..db7c3a0deb7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticatorfactory" "k8s.io/apiserver/pkg/authentication/request/headerrequest" "k8s.io/apiserver/pkg/server" @@ -36,6 +37,17 @@ import ( openapicommon "k8s.io/kube-openapi/pkg/common" ) +// DefaultAuthWebhookRetryBackoff is the default backoff parameters for +// both authentication and authorization webhook used by the apiserver. +func DefaultAuthWebhookRetryBackoff() *wait.Backoff { + return &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + } +} + type RequestHeaderAuthenticationOptions struct { // ClientCAFile is the root certificate bundle to verify client certificates on incoming requests // before trusting usernames in headers. @@ -177,6 +189,11 @@ type DelegatingAuthenticationOptions struct { // TolerateInClusterLookupFailure indicates failures to look up authentication configuration from the cluster configmap should not be fatal. // Setting this can result in an authenticator that will reject all requests. TolerateInClusterLookupFailure bool + + // WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff } func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions { @@ -189,13 +206,23 @@ func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions { GroupHeaders: []string{"x-remote-group"}, ExtraHeaderPrefixes: []string{"x-remote-extra-"}, }, + WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(), } } +// WithCustomRetryBackoff sets the custom backoff parameters for the authentication webhook retry logic. +func (s *DelegatingAuthenticationOptions) WithCustomRetryBackoff(backoff wait.Backoff) { + s.WebhookRetryBackoff = &backoff +} + func (s *DelegatingAuthenticationOptions) Validate() []error { allErrors := []error{} allErrors = append(allErrors, s.RequestHeader.Validate()...) + if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 { + allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps)) + } + return allErrors } @@ -233,8 +260,9 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut } cfg := authenticatorfactory.DelegatingAuthenticatorConfig{ - Anonymous: true, - CacheTTL: s.CacheTTL, + Anonymous: true, + CacheTTL: s.CacheTTL, + WebhookRetryBackoff: s.WebhookRetryBackoff, } client, err := s.getClient() diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go index 818228954e6..04523e8f29a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/pflag" "k8s.io/klog/v2" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" "k8s.io/apiserver/pkg/authorization/path" @@ -63,14 +64,20 @@ type DelegatingAuthorizationOptions struct { // ClientTimeout specifies a time limit for requests made by SubjectAccessReviews client. // The default value is set to 10 seconds. ClientTimeout time.Duration + + // WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic. + // This allows us to configure the sleep time at each iteration and the maximum number of retries allowed + // before we fail the webhook call in order to limit the fan out that ensues when the system is degraded. + WebhookRetryBackoff *wait.Backoff } func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions { return &DelegatingAuthorizationOptions{ // very low for responsiveness, but high enough to handle storms - AllowCacheTTL: 10 * time.Second, - DenyCacheTTL: 10 * time.Second, - ClientTimeout: 10 * time.Second, + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + ClientTimeout: 10 * time.Second, + WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(), } } @@ -91,8 +98,18 @@ func (s *DelegatingAuthorizationOptions) WithClientTimeout(timeout time.Duration s.ClientTimeout = timeout } +// WithCustomRetryBackoff sets the custom backoff parameters for the authorization webhook retry logic. +func (s *DelegatingAuthorizationOptions) WithCustomRetryBackoff(backoff wait.Backoff) { + s.WebhookRetryBackoff = &backoff +} + func (s *DelegatingAuthorizationOptions) Validate() []error { allErrors := []error{} + + if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 { + allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps)) + } + return allErrors } @@ -159,6 +176,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac SubjectAccessReviewClient: client.AuthorizationV1().SubjectAccessReviews(), AllowCacheTTL: s.AllowCacheTTL, DenyCacheTTL: s.DenyCacheTTL, + WebhookRetryBackoff: s.WebhookRetryBackoff, } delegatedAuthorizer, err := cfg.New() if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD b/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD index 91717516b83..4cdaee3bc87 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD @@ -52,6 +52,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go index 799107e1350..abaade352b0 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go @@ -36,12 +36,23 @@ import ( // timeout of the HTTP request, including reading the response body. const defaultRequestTimeout = 30 * time.Second +// DefaultRetryBackoffWithInitialDelay returns the default backoff parameters for webhook retry from a given initial delay. +// Handy for the client that provides a custom initial delay only. +func DefaultRetryBackoffWithInitialDelay(initialBackoffDelay time.Duration) wait.Backoff { + return wait.Backoff{ + Duration: initialBackoffDelay, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + } +} + // GenericWebhook defines a generic client for webhooks with commonly used capabilities, // such as retry requests. type GenericWebhook struct { - RestClient *rest.RESTClient - InitialBackoff time.Duration - ShouldRetry func(error) bool + RestClient *rest.RESTClient + RetryBackoff wait.Backoff + ShouldRetry func(error) bool } // DefaultShouldRetry is a default implementation for the GenericWebhook ShouldRetry function property. @@ -61,11 +72,11 @@ func DefaultShouldRetry(err error) bool { } // NewGenericWebhook creates a new GenericWebhook from the provided kubeconfig file. -func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) { - return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout, customDial) +func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (*GenericWebhook, error) { + return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, retryBackoff, defaultRequestTimeout, customDial) } -func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) { +func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, retryBackoff wait.Backoff, requestTimeout time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) { for _, groupVersion := range groupVersions { if !scheme.IsVersionRegistered(groupVersion) { return nil, fmt.Errorf("webhook plugin requires enabling extension resource: %s", groupVersion) @@ -102,19 +113,20 @@ func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFact return nil, err } - return &GenericWebhook{restClient, initialBackoff, DefaultShouldRetry}, nil + return &GenericWebhook{restClient, retryBackoff, DefaultShouldRetry}, nil } -// WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when -// it returns an error for which this GenericWebhook's ShouldRetry function returns true, confirming it to -// be retriable. If no ShouldRetry has been defined for the webhook, then the default one is used (DefaultShouldRetry). +// WithExponentialBackoff will retry webhookFn() as specified by the given backoff parameters with exponentially +// increasing backoff when it returns an error for which this GenericWebhook's ShouldRetry function returns true, +// confirming it to be retriable. If no ShouldRetry has been defined for the webhook, +// then the default one is used (DefaultShouldRetry). func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn func() rest.Result) rest.Result { var result rest.Result shouldRetry := g.ShouldRetry if shouldRetry == nil { shouldRetry = DefaultShouldRetry } - WithExponentialBackoff(ctx, g.InitialBackoff, func() error { + WithExponentialBackoff(ctx, g.RetryBackoff, func() error { result = webhookFn() return result.Error() }, shouldRetry) @@ -123,18 +135,11 @@ func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn f // WithExponentialBackoff will retry webhookFn up to 5 times with exponentially increasing backoff when // it returns an error for which shouldRetry returns true, confirming it to be retriable. -func WithExponentialBackoff(ctx context.Context, initialBackoff time.Duration, webhookFn func() error, shouldRetry func(error) bool) error { - backoff := wait.Backoff{ - Duration: initialBackoff, - Factor: 1.5, - Jitter: 0.2, - Steps: 5, - } - +func WithExponentialBackoff(ctx context.Context, retryBackoff wait.Backoff, webhookFn func() error, shouldRetry func(error) bool) error { // having a webhook error allows us to track the last actual webhook error for requests that // are later cancelled or time out. var webhookErr error - err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) { + err := wait.ExponentialBackoffWithContext(ctx, retryBackoff, func() (bool, error) { webhookErr = webhookFn() if shouldRetry(webhookErr) { return false, nil diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go index 8d59c76a6c1..c38c3ce7dd2 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go @@ -36,6 +36,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" v1 "k8s.io/client-go/tools/clientcmd/api/v1" @@ -69,7 +70,7 @@ var ( Name: "test-cluster", } groupVersions = []schema.GroupVersion{} - retryBackoff = time.Duration(500) * time.Millisecond + retryBackoff = DefaultRetryBackoffWithInitialDelay(time.Duration(500) * time.Millisecond) ) // TestKubeConfigFile ensures that a kube config file, regardless of validity, is handled properly @@ -670,7 +671,8 @@ func TestWithExponentialBackoffContextIsAlreadyCanceled(t *testing.T) { cancel() // We don't expect the webhook function to be called since the context is already canceled. - err := WithExponentialBackoff(ctx, time.Millisecond, webhookFunc, alwaysRetry) + retryBackoff := wait.Backoff{Steps: 5} + err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry) errExpected := fmt.Errorf("webhook call failed: %s", context.Canceled) if errExpected.Error() != err.Error() { @@ -699,7 +701,8 @@ func TestWithExponentialBackoffWebhookErrorIsMostImportant(t *testing.T) { } // webhook err has higher priority than ctx error. we expect the webhook error to be returned. - err := WithExponentialBackoff(ctx, time.Millisecond, webhookFunc, alwaysRetry) + retryBackoff := wait.Backoff{Steps: 5} + err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry) if attemptsGot != 1 { t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot) @@ -708,3 +711,56 @@ func TestWithExponentialBackoffWebhookErrorIsMostImportant(t *testing.T) { t.Errorf("expected error: %v, but got: %v", errExpected, err) } } + +func TestWithExponentialBackoffParametersNotSet(t *testing.T) { + alwaysRetry := func(e error) bool { + return true + } + + attemptsGot := 0 + webhookFunc := func() error { + attemptsGot++ + return nil + } + + err := WithExponentialBackoff(context.TODO(), wait.Backoff{}, webhookFunc, alwaysRetry) + + errExpected := fmt.Errorf("webhook call failed: %s", wait.ErrWaitTimeout) + if errExpected.Error() != err.Error() { + t.Errorf("expected error: %v, but got: %v", errExpected, err) + } + if attemptsGot != 0 { + t.Errorf("expected %d webhook attempts, but got: %d", 0, attemptsGot) + } +} + +func TestGenericWebhookWithExponentialBackoff(t *testing.T) { + attemptsPerCallExpected := 5 + webhook := &GenericWebhook{ + RetryBackoff: wait.Backoff{ + Duration: time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: attemptsPerCallExpected, + }, + + ShouldRetry: func(e error) bool { + return true + }, + } + + attemptsGot := 0 + webhookFunc := func() rest.Result { + attemptsGot++ + return rest.Result{} + } + + // number of retries should always be local to each call. + totalAttemptsExpected := attemptsPerCallExpected * 2 + webhook.WithExponentialBackoff(context.TODO(), webhookFunc) + webhook.WithExponentialBackoff(context.TODO(), webhookFunc) + + if totalAttemptsExpected != attemptsGot { + t.Errorf("expected a total of %d webhook attempts but got: %d", totalAttemptsExpected, attemptsGot) + } +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD index f73bd537467..9d6a2b88cb9 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD @@ -14,6 +14,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", @@ -32,6 +33,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/install:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go index e75052ee352..0a2aa707879 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/apis/audit/install" "k8s.io/apiserver/pkg/audit" @@ -36,9 +37,9 @@ const ( // PluginName is the name of this plugin, to be used in help and logs. PluginName = "webhook" - // DefaultInitialBackoff is the default amount of time to wait before + // DefaultInitialBackoffDelay is the default amount of time to wait before // retrying sending audit events through a webhook. - DefaultInitialBackoff = 10 * time.Second + DefaultInitialBackoffDelay = 10 * time.Second ) func init() { @@ -61,9 +62,9 @@ func retryOnError(err error) bool { return false } -func loadWebhook(configFile string, groupVersion schema.GroupVersion, initialBackoff time.Duration, customDial utilnet.DialFunc) (*webhook.GenericWebhook, error) { +func loadWebhook(configFile string, groupVersion schema.GroupVersion, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (*webhook.GenericWebhook, error) { w, err := webhook.NewGenericWebhook(audit.Scheme, audit.Codecs, configFile, - []schema.GroupVersion{groupVersion}, initialBackoff, customDial) + []schema.GroupVersion{groupVersion}, retryBackoff, customDial) if err != nil { return nil, err } @@ -79,20 +80,20 @@ type backend struct { // NewDynamicBackend returns an audit backend configured from a REST client that // sends events over HTTP to an external service. -func NewDynamicBackend(rc *rest.RESTClient, initialBackoff time.Duration) audit.Backend { +func NewDynamicBackend(rc *rest.RESTClient, retryBackoff wait.Backoff) audit.Backend { return &backend{ w: &webhook.GenericWebhook{ - RestClient: rc, - InitialBackoff: initialBackoff, - ShouldRetry: retryOnError, + RestClient: rc, + RetryBackoff: retryBackoff, + ShouldRetry: retryOnError, }, name: fmt.Sprintf("dynamic_%s", PluginName), } } // NewBackend returns an audit backend that sends events over HTTP to an external service. -func NewBackend(kubeConfigFile string, groupVersion schema.GroupVersion, initialBackoff time.Duration, customDial utilnet.DialFunc) (audit.Backend, error) { - w, err := loadWebhook(kubeConfigFile, groupVersion, initialBackoff, customDial) +func NewBackend(kubeConfigFile string, groupVersion schema.GroupVersion, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (audit.Backend, error) { + w, err := loadWebhook(kubeConfigFile, groupVersion, retryBackoff, customDial) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_test.go index 9c23589945c..00d8c182d1f 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_test.go @@ -26,6 +26,7 @@ import ( "os" "reflect" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/apimachinery/pkg/util/wait" auditinternal "k8s.io/apiserver/pkg/apis/audit" auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1" @@ -106,7 +108,13 @@ func newWebhook(t *testing.T, endpoint string, groupVersion schema.GroupVersion) // NOTE(ericchiang): Do we need to use a proper serializer? require.NoError(t, stdjson.NewEncoder(f).Encode(config), "writing kubeconfig") - b, err := NewBackend(f.Name(), groupVersion, DefaultInitialBackoff, nil) + retryBackoff := wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + } + b, err := NewBackend(f.Name(), groupVersion, retryBackoff, nil) require.NoError(t, err, "initializing backend") return b.(*backend) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD index 76b89d97962..823ce969a80 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD @@ -20,6 +20,7 @@ go_test( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", @@ -40,6 +41,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go index b04c6a21544..d4bf1b45a91 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/util/webhook" @@ -37,7 +38,11 @@ import ( "k8s.io/klog/v2" ) -const retryBackoff = 500 * time.Millisecond +// DefaultRetryBackoff returns the default backoff parameters for webhook retry. +func DefaultRetryBackoff() *wait.Backoff { + backoff := webhook.DefaultRetryBackoffWithInitialDelay(500 * time.Millisecond) + return &backoff +} // Ensure WebhookTokenAuthenticator implements the authenticator.Token interface. var _ authenticator.Token = (*WebhookTokenAuthenticator)(nil) @@ -47,16 +52,16 @@ type tokenReviewer interface { } type WebhookTokenAuthenticator struct { - tokenReview tokenReviewer - initialBackoff time.Duration - implicitAuds authenticator.Audiences + tokenReview tokenReviewer + retryBackoff wait.Backoff + implicitAuds authenticator.Audiences } // NewFromInterface creates a webhook authenticator using the given tokenReview // client. It is recommend to wrap this authenticator with the token cache // authenticator implemented in // k8s.io/apiserver/pkg/authentication/token/cache. -func NewFromInterface(tokenReview authenticationv1client.TokenReviewInterface, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { +func NewFromInterface(tokenReview authenticationv1client.TokenReviewInterface, implicitAuds authenticator.Audiences, retryBackoff wait.Backoff) (*WebhookTokenAuthenticator, error) { return newWithBackoff(tokenReview, retryBackoff, implicitAuds) } @@ -64,8 +69,8 @@ func NewFromInterface(tokenReview authenticationv1client.TokenReviewInterface, i // file. It is recommend to wrap this authenticator with the token cache // authenticator implemented in // k8s.io/apiserver/pkg/authentication/token/cache. -func New(kubeConfigFile string, version string, implicitAuds authenticator.Audiences, customDial utilnet.DialFunc) (*WebhookTokenAuthenticator, error) { - tokenReview, err := tokenReviewInterfaceFromKubeconfig(kubeConfigFile, version, customDial) +func New(kubeConfigFile string, version string, implicitAuds authenticator.Audiences, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (*WebhookTokenAuthenticator, error) { + tokenReview, err := tokenReviewInterfaceFromKubeconfig(kubeConfigFile, version, retryBackoff, customDial) if err != nil { return nil, err } @@ -73,8 +78,8 @@ func New(kubeConfigFile string, version string, implicitAuds authenticator.Audie } // newWithBackoff allows tests to skip the sleep. -func newWithBackoff(tokenReview tokenReviewer, initialBackoff time.Duration, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { - return &WebhookTokenAuthenticator{tokenReview, initialBackoff, implicitAuds}, nil +func newWithBackoff(tokenReview tokenReviewer, retryBackoff wait.Backoff, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { + return &WebhookTokenAuthenticator{tokenReview, retryBackoff, implicitAuds}, nil } // AuthenticateToken implements the authenticator.Token interface. @@ -102,7 +107,7 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token err error auds authenticator.Audiences ) - webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error { + webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error { result, err = w.tokenReview.Create(ctx, r, metav1.CreateOptions{}) return err }, webhook.DefaultShouldRetry) @@ -154,7 +159,7 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token // tokenReviewInterfaceFromKubeconfig builds a client from the specified kubeconfig file, // and returns a TokenReviewInterface that uses that client. Note that the client submits TokenReview // requests to the exact path specified in the kubeconfig file, so arbitrary non-API servers can be targeted. -func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, customDial utilnet.DialFunc) (tokenReviewer, error) { +func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (tokenReviewer, error) { localScheme := runtime.NewScheme() if err := scheme.AddToScheme(localScheme); err != nil { return nil, err @@ -166,7 +171,7 @@ func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, c if err := localScheme.SetVersionPriority(groupVersions...); err != nil { return nil, err } - gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0, customDial) + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff, customDial) if err != nil { return nil, err } @@ -177,7 +182,7 @@ func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, c if err := localScheme.SetVersionPriority(groupVersions...); err != nil { return nil, err } - gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0, customDial) + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff, customDial) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1_test.go index e9bf2a36d7b..913c5417050 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1_test.go @@ -33,12 +33,20 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/token/cache" "k8s.io/apiserver/pkg/authentication/user" v1 "k8s.io/client-go/tools/clientcmd/api/v1" ) +var testRetryBackoff = wait.Backoff{ + Duration: 5 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, +} + // V1Service mocks a remote authentication service. type V1Service interface { // Review looks at the TokenReviewSpec and provides an authentication @@ -193,12 +201,12 @@ func newV1TokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, return nil, err } - c, err := tokenReviewInterfaceFromKubeconfig(p, "v1", nil) + c, err := tokenReviewInterfaceFromKubeconfig(p, "v1", testRetryBackoff, nil) if err != nil { return nil, err } - authn, err := newWithBackoff(c, 0, implicitAuds) + authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1beta1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1beta1_test.go index e5ea01baee9..93e1078e461 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1beta1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_v1beta1_test.go @@ -195,12 +195,12 @@ func newV1beta1TokenAuthenticator(serverURL string, clientCert, clientKey, ca [] return nil, err } - c, err := tokenReviewInterfaceFromKubeconfig(p, "v1beta1", nil) + c, err := tokenReviewInterfaceFromKubeconfig(p, "v1beta1", testRetryBackoff, nil) if err != nil { return nil, err } - authn, err := newWithBackoff(c, 0, implicitAuds) + authn, err := newWithBackoff(c, testRetryBackoff, implicitAuds) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD index 4b31b4dd459..a45057f4d3e 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD @@ -20,6 +20,7 @@ go_test( "//staging/src/k8s.io/api/authorization/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library", @@ -40,6 +41,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/cache:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go index d7f4f631ec9..5c9f28ad40c 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/cache" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/util/webhook" @@ -40,11 +41,16 @@ import ( ) const ( - retryBackoff = 500 * time.Millisecond // The maximum length of requester-controlled attributes to allow caching. maxControlledAttrCacheSize = 10000 ) +// DefaultRetryBackoff returns the default backoff parameters for webhook retry. +func DefaultRetryBackoff() *wait.Backoff { + backoff := webhook.DefaultRetryBackoffWithInitialDelay(500 * time.Millisecond) + return &backoff +} + // Ensure Webhook implements the authorizer.Authorizer interface. var _ authorizer.Authorizer = (*WebhookAuthorizer)(nil) @@ -57,12 +63,12 @@ type WebhookAuthorizer struct { responseCache *cache.LRUExpireCache authorizedTTL time.Duration unauthorizedTTL time.Duration - initialBackoff time.Duration + retryBackoff wait.Backoff decisionOnError authorizer.Decision } // NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client -func NewFromInterface(subjectAccessReview authorizationv1client.SubjectAccessReviewInterface, authorizedTTL, unauthorizedTTL time.Duration) (*WebhookAuthorizer, error) { +func NewFromInterface(subjectAccessReview authorizationv1client.SubjectAccessReviewInterface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff) (*WebhookAuthorizer, error) { return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff) } @@ -85,8 +91,8 @@ func NewFromInterface(subjectAccessReview authorizationv1client.SubjectAccessRev // // For additional HTTP configuration, refer to the kubeconfig documentation // https://kubernetes.io/docs/user-guide/kubeconfig-file/. -func New(kubeConfigFile string, version string, authorizedTTL, unauthorizedTTL time.Duration, customDial utilnet.DialFunc) (*WebhookAuthorizer, error) { - subjectAccessReview, err := subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile, version, customDial) +func New(kubeConfigFile string, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (*WebhookAuthorizer, error) { + subjectAccessReview, err := subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile, version, retryBackoff, customDial) if err != nil { return nil, err } @@ -94,13 +100,13 @@ func New(kubeConfigFile string, version string, authorizedTTL, unauthorizedTTL t } // newWithBackoff allows tests to skip the sleep. -func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL, initialBackoff time.Duration) (*WebhookAuthorizer, error) { +func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff) (*WebhookAuthorizer, error) { return &WebhookAuthorizer{ subjectAccessReview: subjectAccessReview, responseCache: cache.NewLRUExpireCache(8192), authorizedTTL: authorizedTTL, unauthorizedTTL: unauthorizedTTL, - initialBackoff: initialBackoff, + retryBackoff: retryBackoff, decisionOnError: authorizer.DecisionNoOpinion, }, nil } @@ -190,7 +196,7 @@ func (w *WebhookAuthorizer) Authorize(ctx context.Context, attr authorizer.Attri result *authorizationv1.SubjectAccessReview err error ) - webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error { + webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error { result, err = w.subjectAccessReview.Create(ctx, r, metav1.CreateOptions{}) return err }, webhook.DefaultShouldRetry) @@ -246,7 +252,7 @@ func convertToSARExtra(extra map[string][]string) map[string]authorizationv1.Ext // subjectAccessReviewInterfaceFromKubeconfig builds a client from the specified kubeconfig file, // and returns a SubjectAccessReviewInterface that uses that client. Note that the client submits SubjectAccessReview // requests to the exact path specified in the kubeconfig file, so arbitrary non-API servers can be targeted. -func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, customDial utilnet.DialFunc) (subjectAccessReviewer, error) { +func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version string, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (subjectAccessReviewer, error) { localScheme := runtime.NewScheme() if err := scheme.AddToScheme(localScheme); err != nil { return nil, err @@ -258,7 +264,7 @@ func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version s if err := localScheme.SetVersionPriority(groupVersions...); err != nil { return nil, err } - gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0, customDial) + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff, customDial) if err != nil { return nil, err } @@ -269,7 +275,7 @@ func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string, version s if err := localScheme.SetVersionPriority(groupVersions...); err != nil { return nil, err } - gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0, customDial) + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff, customDial) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go index f1ccc84d7b2..8426b4fa2d3 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1_test.go @@ -37,11 +37,19 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" v1 "k8s.io/client-go/tools/clientcmd/api/v1" ) +var testRetryBackoff = wait.Backoff{ + Duration: 5 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, +} + func TestV1NewFromConfig(t *testing.T) { dir, err := ioutil.TempDir("", "") if err != nil { @@ -186,11 +194,11 @@ current-context: default return fmt.Errorf("failed to execute test template: %v", err) } // Create a new authorizer - sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1", nil) + sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1", testRetryBackoff, nil) if err != nil { return fmt.Errorf("error building sar client: %v", err) } - _, err = newWithBackoff(sarClient, 0, 0, 0) + _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff) return err }() if err != nil && !tt.wantErr { @@ -325,11 +333,11 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache if err := json.NewEncoder(tempfile).Encode(config); err != nil { return nil, err } - sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1", nil) + sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1", testRetryBackoff, nil) if err != nil { return nil, fmt.Errorf("error building sar client: %v", err) } - return newWithBackoff(sarClient, cacheTime, cacheTime, 0) + return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff) } func TestV1TLSConfig(t *testing.T) { diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go index 7d664883d75..db207c73f08 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go @@ -186,11 +186,11 @@ current-context: default return fmt.Errorf("failed to execute test template: %v", err) } // Create a new authorizer - sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1beta1", nil) + sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1beta1", testRetryBackoff, nil) if err != nil { return fmt.Errorf("error building sar client: %v", err) } - _, err = newWithBackoff(sarClient, 0, 0, 0) + _, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff) return err }() if err != nil && !tt.wantErr { @@ -325,11 +325,11 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, if err := json.NewEncoder(tempfile).Encode(config); err != nil { return nil, err } - sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1beta1", nil) + sarClient, err := subjectAccessReviewInterfaceFromKubeconfig(p, "v1beta1", testRetryBackoff, nil) if err != nil { return nil, fmt.Errorf("error building sar client: %v", err) } - return newWithBackoff(sarClient, cacheTime, cacheTime, 0) + return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff) } func TestV1beta1TLSConfig(t *testing.T) { diff --git a/staging/src/k8s.io/cloud-provider/options/options_test.go b/staging/src/k8s.io/cloud-provider/options/options_test.go index 7eeb5a98334..33e7d5db0a3 100644 --- a/staging/src/k8s.io/cloud-provider/options/options_test.go +++ b/staging/src/k8s.io/cloud-provider/options/options_test.go @@ -103,8 +103,9 @@ func TestDefaultFlags(t *testing.T) { BindNetwork: "tcp", }).WithLoopback(), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + CacheTTL: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ UsernameHeaders: []string{"x-remote-user"}, GroupHeaders: []string{"x-remote-group"}, @@ -116,6 +117,7 @@ func TestDefaultFlags(t *testing.T) { AllowCacheTTL: 10 * time.Second, DenyCacheTTL: 10 * time.Second, ClientTimeout: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), RemoteKubeConfigFileOptional: true, AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or }, @@ -236,8 +238,9 @@ func TestAddFlags(t *testing.T) { BindNetwork: "tcp", }).WithLoopback(), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + CacheTTL: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ UsernameHeaders: []string{"x-remote-user"}, GroupHeaders: []string{"x-remote-group"}, @@ -249,6 +252,7 @@ func TestAddFlags(t *testing.T) { AllowCacheTTL: 10 * time.Second, DenyCacheTTL: 10 * time.Second, ClientTimeout: 10 * time.Second, + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), RemoteKubeConfigFileOptional: true, AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or }, diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go index 421eef801c6..4cbf2b02a5e 100644 --- a/test/integration/auth/auth_test.go +++ b/test/integration/auth/auth_test.go @@ -38,6 +38,7 @@ import ( authenticationv1beta1 "k8s.io/api/authentication/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/group" "k8s.io/apiserver/pkg/authentication/request/bearertoken" @@ -86,7 +87,14 @@ func getTestWebhookTokenAuth(serverURL string, customDial utilnet.DialFunc) (aut if err := json.NewEncoder(kubecfgFile).Encode(config); err != nil { return nil, err } - webhookTokenAuth, err := webhook.New(kubecfgFile.Name(), "v1beta1", nil, customDial) + + retryBackoff := wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + } + webhookTokenAuth, err := webhook.New(kubecfgFile.Name(), "v1beta1", nil, retryBackoff, customDial) if err != nil { return nil, err }