Merge pull request #97820 from tkashem/webhook-error

prevent panic on webhook authenticator and authorizer timeout before response
This commit is contained in:
Kubernetes Prow Robot 2021-01-08 10:34:41 -08:00 committed by GitHub
commit cfc96c7db7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 16 deletions

View File

@ -711,6 +711,33 @@ func TestWithExponentialBackoffWebhookErrorIsMostImportant(t *testing.T) {
} }
} }
func TestWithExponentialBackoffWithRetryExhaustedWhileContextIsNotCanceled(t *testing.T) {
alwaysRetry := func(e error) bool {
return true
}
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
attemptsGot := 0
errExpected := errors.New("webhook not available")
webhookFunc := func() error {
attemptsGot++
return errExpected
}
// webhook err has higher priority than ctx error. we expect the webhook error to be returned.
retryBackoff := wait.Backoff{Steps: 5}
err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry)
if attemptsGot != 5 {
t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot)
}
if errExpected != err {
t.Errorf("expected error: %v, but got: %v", errExpected, err)
}
}
func TestWithExponentialBackoffParametersNotSet(t *testing.T) { func TestWithExponentialBackoffParametersNotSet(t *testing.T) {
alwaysRetry := func(e error) bool { alwaysRetry := func(e error) bool {
return true return true

View File

@ -104,14 +104,14 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token
} }
var ( var (
result *authenticationv1.TokenReview result *authenticationv1.TokenReview
err error
auds authenticator.Audiences auds authenticator.Audiences
) )
webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error { // WithExponentialBackoff will return tokenreview create error (tokenReviewErr) if any.
result, err = w.tokenReview.Create(ctx, r, metav1.CreateOptions{}) if err := webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error {
return err var tokenReviewErr error
}, webhook.DefaultShouldRetry) result, tokenReviewErr = w.tokenReview.Create(ctx, r, metav1.CreateOptions{})
if err != nil { return tokenReviewErr
}, webhook.DefaultShouldRetry); err != nil {
// An error here indicates bad configuration or an outage. Log for debugging. // An error here indicates bad configuration or an outage. Log for debugging.
klog.Errorf("Failed to make webhook authenticator request: %v", err) klog.Errorf("Failed to make webhook authenticator request: %v", err)
return nil, false, err return nil, false, err

View File

@ -192,19 +192,17 @@ func (w *WebhookAuthorizer) Authorize(ctx context.Context, attr authorizer.Attri
if entry, ok := w.responseCache.Get(string(key)); ok { if entry, ok := w.responseCache.Get(string(key)); ok {
r.Status = entry.(authorizationv1.SubjectAccessReviewStatus) r.Status = entry.(authorizationv1.SubjectAccessReviewStatus)
} else { } else {
var ( var result *authorizationv1.SubjectAccessReview
result *authorizationv1.SubjectAccessReview // WithExponentialBackoff will return SAR create error (sarErr) if any.
err error if err := webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error {
) var sarErr error
webhook.WithExponentialBackoff(ctx, w.retryBackoff, func() error { result, sarErr = w.subjectAccessReview.Create(ctx, r, metav1.CreateOptions{})
result, err = w.subjectAccessReview.Create(ctx, r, metav1.CreateOptions{}) return sarErr
return err }, webhook.DefaultShouldRetry); err != nil {
}, webhook.DefaultShouldRetry)
if err != nil {
// An error here indicates bad configuration or an outage. Log for debugging.
klog.Errorf("Failed to make webhook authorizer request: %v", err) klog.Errorf("Failed to make webhook authorizer request: %v", err)
return w.decisionOnError, "", err return w.decisionOnError, "", err
} }
r.Status = result.Status r.Status = result.Status
if shouldCache(attr) { if shouldCache(attr) {
if r.Status.Allowed { if r.Status.Allowed {