Propagate context to ExponentialBackoff

This commit is contained in:
Jordan Liggitt 2019-09-24 09:43:04 -04:00
parent 92eb072989
commit 4c686ddc1c
6 changed files with 20 additions and 11 deletions

View File

@ -160,13 +160,13 @@ func (a *Plugin) Validate(ctx context.Context, attributes admission.Attributes,
Namespace: attributes.GetNamespace(), Namespace: attributes.GetNamespace(),
}, },
} }
if err := a.admitPod(pod, attributes, &imageReview); err != nil { if err := a.admitPod(ctx, pod, attributes, &imageReview); err != nil {
return admission.NewForbidden(attributes, err) return admission.NewForbidden(attributes, err)
} }
return nil return nil
} }
func (a *Plugin) admitPod(pod *api.Pod, attributes admission.Attributes, review *v1alpha1.ImageReview) error { func (a *Plugin) admitPod(ctx context.Context, pod *api.Pod, attributes admission.Attributes, review *v1alpha1.ImageReview) error {
cacheKey, err := json.Marshal(review.Spec) cacheKey, err := json.Marshal(review.Spec)
if err != nil { if err != nil {
return err return err
@ -174,7 +174,7 @@ func (a *Plugin) admitPod(pod *api.Pod, attributes admission.Attributes, review
if entry, ok := a.responseCache.Get(string(cacheKey)); ok { if entry, ok := a.responseCache.Get(string(cacheKey)); ok {
review.Status = entry.(v1alpha1.ImageReviewStatus) review.Status = entry.(v1alpha1.ImageReviewStatus)
} else { } else {
result := a.webhook.WithExponentialBackoff(func() rest.Result { result := a.webhook.WithExponentialBackoff(ctx, func() rest.Result {
return a.webhook.RestClient.Post().Body(review).Do() return a.webhook.RestClient.Post().Body(review).Do()
}) })

View File

@ -18,6 +18,7 @@ limitations under the License.
package webhook package webhook
import ( import (
"context"
"fmt" "fmt"
"time" "time"
@ -81,9 +82,9 @@ func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFact
// WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when // WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when
// it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true. // it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true.
func (g *GenericWebhook) WithExponentialBackoff(webhookFn func() rest.Result) rest.Result { func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn func() rest.Result) rest.Result {
var result rest.Result var result rest.Result
WithExponentialBackoff(g.InitialBackoff, func() error { WithExponentialBackoff(ctx, g.InitialBackoff, func() error {
result = webhookFn() result = webhookFn()
return result.Error() return result.Error()
}) })
@ -92,7 +93,7 @@ func (g *GenericWebhook) WithExponentialBackoff(webhookFn func() rest.Result) re
// WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when // WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when
// it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true. // it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true.
func WithExponentialBackoff(initialBackoff time.Duration, webhookFn func() error) error { func WithExponentialBackoff(ctx context.Context, initialBackoff time.Duration, webhookFn func() error) error {
backoff := wait.Backoff{ backoff := wait.Backoff{
Duration: initialBackoff, Duration: initialBackoff,
Factor: 1.5, Factor: 1.5,
@ -103,6 +104,12 @@ func WithExponentialBackoff(initialBackoff time.Duration, webhookFn func() error
var err error var err error
wait.ExponentialBackoff(backoff, func() (bool, error) { wait.ExponentialBackoff(backoff, func() (bool, error) {
err = webhookFn() err = webhookFn()
if ctx.Err() != nil {
// we timed out or were cancelled, we should not retry
return true, err
}
// these errors indicate a transient error that should be retried. // these errors indicate a transient error that should be retried.
if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) { if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) {
return false, nil return false, nil

View File

@ -17,6 +17,7 @@ limitations under the License.
package webhook package webhook
import ( import (
"context"
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"encoding/json" "encoding/json"
@ -550,7 +551,7 @@ func TestWithExponentialBackoff(t *testing.T) {
t.Fatalf("failed to create the webhook: %v", err) t.Fatalf("failed to create the webhook: %v", err)
} }
result := wh.WithExponentialBackoff(func() rest.Result { result := wh.WithExponentialBackoff(context.Background(), func() rest.Result {
return wh.RestClient.Get().Do() return wh.RestClient.Get().Do()
}) })
@ -562,7 +563,7 @@ func TestWithExponentialBackoff(t *testing.T) {
t.Errorf("unexpected status code: %d", statusCode) t.Errorf("unexpected status code: %d", statusCode)
} }
result = wh.WithExponentialBackoff(func() rest.Result { result = wh.WithExponentialBackoff(context.Background(), func() rest.Result {
return wh.RestClient.Get().Do() return wh.RestClient.Get().Do()
}) })

View File

@ -18,6 +18,7 @@ limitations under the License.
package webhook package webhook
import ( import (
"context"
"fmt" "fmt"
"time" "time"
@ -95,7 +96,7 @@ func (b *backend) processEvents(ev ...*auditinternal.Event) error {
for _, e := range ev { for _, e := range ev {
list.Items = append(list.Items, *e) list.Items = append(list.Items, *e)
} }
return b.w.WithExponentialBackoff(func() rest.Result { return b.w.WithExponentialBackoff(context.Background(), func() rest.Result {
trace := utiltrace.New("Call Audit Events webhook", trace := utiltrace.New("Call Audit Events webhook",
utiltrace.Field{"name", b.name}, utiltrace.Field{"name", b.name},
utiltrace.Field{"event-count", len(list.Items)}) utiltrace.Field{"event-count", len(list.Items)})

View File

@ -98,7 +98,7 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token
err error err error
auds authenticator.Audiences auds authenticator.Audiences
) )
webhook.WithExponentialBackoff(w.initialBackoff, func() error { webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error {
result, err = w.tokenReview.Create(r) result, err = w.tokenReview.Create(r)
return err return err
}) })

View File

@ -188,7 +188,7 @@ func (w *WebhookAuthorizer) Authorize(ctx context.Context, attr authorizer.Attri
result *authorization.SubjectAccessReview result *authorization.SubjectAccessReview
err error err error
) )
webhook.WithExponentialBackoff(w.initialBackoff, func() error { webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error {
result, err = w.subjectAccessReview.Create(r) result, err = w.subjectAccessReview.Create(r)
return err return err
}) })