From b15a1943d51adfb8c5e0185d58d25e038c3d6ade Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 2 Sep 2024 20:18:47 +0200 Subject: [PATCH] client-go/rest: backoff with context support The BackoffManager interface sleeps without considering the caller's context, i.e. cancellation is not supported. This alone is reason enough to deprecate it and to replace it with an interface that supports a context parameter. The other reason is that contextual logging needs that parameter. --- .../src/k8s.io/apiextensions-apiserver/go.sum | 1 + staging/src/k8s.io/apiserver/go.sum | 1 + staging/src/k8s.io/client-go/go.mod | 1 + staging/src/k8s.io/client-go/go.sum | 1 + .../src/k8s.io/client-go/rest/.mockery.yaml | 10 ++ staging/src/k8s.io/client-go/rest/client.go | 4 +- .../src/k8s.io/client-go/rest/client_test.go | 21 +-- .../rest/mock_backoff_manager_test.go | 168 ++++++++++++++++++ staging/src/k8s.io/client-go/rest/request.go | 20 ++- .../src/k8s.io/client-go/rest/request_test.go | 78 ++++++-- .../src/k8s.io/client-go/rest/urlbackoff.go | 101 +++++++++-- .../k8s.io/client-go/rest/urlbackoff_test.go | 39 ++++ .../src/k8s.io/client-go/rest/with_retry.go | 10 +- staging/src/k8s.io/cloud-provider/go.sum | 1 + .../src/k8s.io/code-generator/examples/go.sum | 2 + staging/src/k8s.io/component-base/go.sum | 1 + staging/src/k8s.io/component-helpers/go.sum | 1 + staging/src/k8s.io/controller-manager/go.sum | 1 + .../k8s.io/dynamic-resource-allocation/go.sum | 1 + staging/src/k8s.io/endpointslice/go.sum | 1 + staging/src/k8s.io/kube-aggregator/go.sum | 1 + staging/src/k8s.io/kubelet/go.sum | 1 + staging/src/k8s.io/metrics/go.sum | 1 + .../src/k8s.io/pod-security-admission/go.sum | 1 + staging/src/k8s.io/sample-apiserver/go.sum | 1 + staging/src/k8s.io/sample-controller/go.sum | 2 + 26 files changed, 419 insertions(+), 51 deletions(-) create mode 100644 staging/src/k8s.io/client-go/rest/.mockery.yaml create mode 100644 staging/src/k8s.io/client-go/rest/mock_backoff_manager_test.go diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index 7417687d967..3b6f384cf35 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -318,6 +318,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index 9752ca6b795..7290f614182 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -320,6 +320,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/staging/src/k8s.io/client-go/go.mod b/staging/src/k8s.io/client-go/go.mod index ba13f033799..3a50ca0981c 100644 --- a/staging/src/k8s.io/client-go/go.mod +++ b/staging/src/k8s.io/client-go/go.mod @@ -58,6 +58,7 @@ require ( github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/staging/src/k8s.io/client-go/go.sum b/staging/src/k8s.io/client-go/go.sum index 0582468abd1..d04ea0816f7 100644 --- a/staging/src/k8s.io/client-go/go.sum +++ b/staging/src/k8s.io/client-go/go.sum @@ -88,6 +88,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/client-go/rest/.mockery.yaml b/staging/src/k8s.io/client-go/rest/.mockery.yaml new file mode 100644 index 00000000000..e21d7b5be26 --- /dev/null +++ b/staging/src/k8s.io/client-go/rest/.mockery.yaml @@ -0,0 +1,10 @@ +--- +dir: . +filename: "mock_{{.InterfaceName | snakecase}}_test.go" +boilerplate-file: ../../../../../hack/boilerplate/boilerplate.generatego.txt +outpkg: rest +with-expecter: true +packages: + k8s.io/client-go/rest: + interfaces: + BackoffManager: diff --git a/staging/src/k8s.io/client-go/rest/client.go b/staging/src/k8s.io/client-go/rest/client.go index 29a254484c4..a085c334f98 100644 --- a/staging/src/k8s.io/client-go/rest/client.go +++ b/staging/src/k8s.io/client-go/rest/client.go @@ -93,7 +93,7 @@ type RESTClient struct { content requestClientContentConfigProvider // creates BackoffManager that is passed to requests. - createBackoffMgr func() BackoffManager + createBackoffMgr func() BackoffManagerWithContext // rateLimiter is shared among all requests created by this client unless specifically // overridden. @@ -178,7 +178,7 @@ func (c *RESTClient) GetRateLimiter() flowcontrol.RateLimiter { // readExpBackoffConfig handles the internal logic of determining what the // backoff policy is. By default if no information is available, NoBackoff. // TODO Generalize this see #17727 . -func readExpBackoffConfig() BackoffManager { +func readExpBackoffConfig() BackoffManagerWithContext { backoffBase := os.Getenv(envBackoffBase) backoffDuration := os.Getenv(envBackoffDuration) diff --git a/staging/src/k8s.io/client-go/rest/client_test.go b/staging/src/k8s.io/client-go/rest/client_test.go index ebb35c50900..c03f6832c69 100644 --- a/staging/src/k8s.io/client-go/rest/client_test.go +++ b/staging/src/k8s.io/client-go/rest/client_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" utiltesting "k8s.io/client-go/util/testing" + "k8s.io/klog/v2/ktesting" "github.com/google/go-cmp/cmp" ) @@ -335,26 +336,26 @@ func TestHTTPProxy(t *testing.T) { } func TestCreateBackoffManager(t *testing.T) { - + _, ctx := ktesting.NewTestContext(t) theUrl, _ := url.Parse("http://localhost") // 1 second base backoff + duration of 2 seconds -> exponential backoff for requests. t.Setenv(envBackoffBase, "1") t.Setenv(envBackoffDuration, "2") backoff := readExpBackoffConfig() - backoff.UpdateBackoff(theUrl, nil, 500) - backoff.UpdateBackoff(theUrl, nil, 500) - if backoff.CalculateBackoff(theUrl)/time.Second != 2 { + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) + if backoff.CalculateBackoffWithContext(ctx, theUrl)/time.Second != 2 { t.Errorf("Backoff env not working.") } // 0 duration -> no backoff. t.Setenv(envBackoffBase, "1") t.Setenv(envBackoffDuration, "0") - backoff.UpdateBackoff(theUrl, nil, 500) - backoff.UpdateBackoff(theUrl, nil, 500) + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) backoff = readExpBackoffConfig() - if backoff.CalculateBackoff(theUrl)/time.Second != 0 { + if backoff.CalculateBackoffWithContext(ctx, theUrl)/time.Second != 0 { t.Errorf("Zero backoff duration, but backoff still occurring.") } @@ -362,9 +363,9 @@ func TestCreateBackoffManager(t *testing.T) { t.Setenv(envBackoffBase, "") t.Setenv(envBackoffDuration, "") backoff = readExpBackoffConfig() - backoff.UpdateBackoff(theUrl, nil, 500) - backoff.UpdateBackoff(theUrl, nil, 500) - if backoff.CalculateBackoff(theUrl)/time.Second != 0 { + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) + backoff.UpdateBackoffWithContext(ctx, theUrl, nil, 500) + if backoff.CalculateBackoffWithContext(ctx, theUrl)/time.Second != 0 { t.Errorf("Backoff should have been 0.") } diff --git a/staging/src/k8s.io/client-go/rest/mock_backoff_manager_test.go b/staging/src/k8s.io/client-go/rest/mock_backoff_manager_test.go new file mode 100644 index 00000000000..3cd4585ae9e --- /dev/null +++ b/staging/src/k8s.io/client-go/rest/mock_backoff_manager_test.go @@ -0,0 +1,168 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by mockery v2.40.3. DO NOT EDIT. + +package rest + +import ( + mock "github.com/stretchr/testify/mock" + + time "time" + + url "net/url" +) + +// MockBackoffManager is an autogenerated mock type for the BackoffManager type +type MockBackoffManager struct { + mock.Mock +} + +type MockBackoffManager_Expecter struct { + mock *mock.Mock +} + +func (_m *MockBackoffManager) EXPECT() *MockBackoffManager_Expecter { + return &MockBackoffManager_Expecter{mock: &_m.Mock} +} + +// CalculateBackoff provides a mock function with given fields: actualURL +func (_m *MockBackoffManager) CalculateBackoff(actualURL *url.URL) time.Duration { + ret := _m.Called(actualURL) + + if len(ret) == 0 { + panic("no return value specified for CalculateBackoff") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func(*url.URL) time.Duration); ok { + r0 = rf(actualURL) + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// MockBackoffManager_CalculateBackoff_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CalculateBackoff' +type MockBackoffManager_CalculateBackoff_Call struct { + *mock.Call +} + +// CalculateBackoff is a helper method to define mock.On call +// - actualURL *url.URL +func (_e *MockBackoffManager_Expecter) CalculateBackoff(actualURL interface{}) *MockBackoffManager_CalculateBackoff_Call { + return &MockBackoffManager_CalculateBackoff_Call{Call: _e.mock.On("CalculateBackoff", actualURL)} +} + +func (_c *MockBackoffManager_CalculateBackoff_Call) Run(run func(actualURL *url.URL)) *MockBackoffManager_CalculateBackoff_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*url.URL)) + }) + return _c +} + +func (_c *MockBackoffManager_CalculateBackoff_Call) Return(_a0 time.Duration) *MockBackoffManager_CalculateBackoff_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockBackoffManager_CalculateBackoff_Call) RunAndReturn(run func(*url.URL) time.Duration) *MockBackoffManager_CalculateBackoff_Call { + _c.Call.Return(run) + return _c +} + +// Sleep provides a mock function with given fields: d +func (_m *MockBackoffManager) Sleep(d time.Duration) { + _m.Called(d) +} + +// MockBackoffManager_Sleep_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Sleep' +type MockBackoffManager_Sleep_Call struct { + *mock.Call +} + +// Sleep is a helper method to define mock.On call +// - d time.Duration +func (_e *MockBackoffManager_Expecter) Sleep(d interface{}) *MockBackoffManager_Sleep_Call { + return &MockBackoffManager_Sleep_Call{Call: _e.mock.On("Sleep", d)} +} + +func (_c *MockBackoffManager_Sleep_Call) Run(run func(d time.Duration)) *MockBackoffManager_Sleep_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(time.Duration)) + }) + return _c +} + +func (_c *MockBackoffManager_Sleep_Call) Return() *MockBackoffManager_Sleep_Call { + _c.Call.Return() + return _c +} + +func (_c *MockBackoffManager_Sleep_Call) RunAndReturn(run func(time.Duration)) *MockBackoffManager_Sleep_Call { + _c.Call.Return(run) + return _c +} + +// UpdateBackoff provides a mock function with given fields: actualURL, err, responseCode +func (_m *MockBackoffManager) UpdateBackoff(actualURL *url.URL, err error, responseCode int) { + _m.Called(actualURL, err, responseCode) +} + +// MockBackoffManager_UpdateBackoff_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateBackoff' +type MockBackoffManager_UpdateBackoff_Call struct { + *mock.Call +} + +// UpdateBackoff is a helper method to define mock.On call +// - actualURL *url.URL +// - err error +// - responseCode int +func (_e *MockBackoffManager_Expecter) UpdateBackoff(actualURL interface{}, err interface{}, responseCode interface{}) *MockBackoffManager_UpdateBackoff_Call { + return &MockBackoffManager_UpdateBackoff_Call{Call: _e.mock.On("UpdateBackoff", actualURL, err, responseCode)} +} + +func (_c *MockBackoffManager_UpdateBackoff_Call) Run(run func(actualURL *url.URL, err error, responseCode int)) *MockBackoffManager_UpdateBackoff_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*url.URL), args[1].(error), args[2].(int)) + }) + return _c +} + +func (_c *MockBackoffManager_UpdateBackoff_Call) Return() *MockBackoffManager_UpdateBackoff_Call { + _c.Call.Return() + return _c +} + +func (_c *MockBackoffManager_UpdateBackoff_Call) RunAndReturn(run func(*url.URL, error, int)) *MockBackoffManager_UpdateBackoff_Call { + _c.Call.Return(run) + return _c +} + +// NewMockBackoffManager creates a new instance of MockBackoffManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockBackoffManager(t interface { + mock.TestingT + Cleanup(func()) +}) *MockBackoffManager { + mock := &MockBackoffManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/staging/src/k8s.io/client-go/rest/request.go b/staging/src/k8s.io/client-go/rest/request.go index b10db0ad8f3..864ccd876e0 100644 --- a/staging/src/k8s.io/client-go/rest/request.go +++ b/staging/src/k8s.io/client-go/rest/request.go @@ -106,7 +106,7 @@ type Request struct { warningHandler WarningHandlerWithContext rateLimiter flowcontrol.RateLimiter - backoff BackoffManager + backoff BackoffManagerWithContext timeout time.Duration maxRetries int @@ -136,7 +136,7 @@ type Request struct { // NewRequest creates a new request helper object for accessing runtime.Objects on a server. func NewRequest(c *RESTClient) *Request { - var backoff BackoffManager + var backoff BackoffManagerWithContext if c.createBackoffMgr != nil { backoff = c.createBackoffMgr() } @@ -259,13 +259,27 @@ func (r *Request) Resource(resource string) *Request { } // BackOff sets the request's backoff manager to the one specified, -// or defaults to the stub implementation if nil is provided +// or defaults to the stub implementation if nil is provided. +// +// Deprecated: BackoffManager.Sleep ignores the caller's context. Use BackOffWithContext and BackoffManagerWithContext instead. func (r *Request) BackOff(manager BackoffManager) *Request { if manager == nil { r.backoff = &NoBackoff{} return r } + r.backoff = &backoffManagerNopContext{BackoffManager: manager} + return r +} + +// BackOffWithContext sets the request's backoff manager to the one specified, +// or defaults to the stub implementation if nil is provided. +func (r *Request) BackOffWithContext(manager BackoffManagerWithContext) *Request { + if manager == nil { + r.backoff = &NoBackoff{} + return r + } + r.backoff = manager return r } diff --git a/staging/src/k8s.io/client-go/rest/request_test.go b/staging/src/k8s.io/client-go/rest/request_test.go index 013a22816ec..0096501bc87 100644 --- a/staging/src/k8s.io/client-go/rest/request_test.go +++ b/staging/src/k8s.io/client-go/rest/request_test.go @@ -1489,6 +1489,7 @@ func TestDoRequestNewWay(t *testing.T) { // This test assumes that the client implementation backs off exponentially, for an individual request. func TestBackoffLifecycle(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) count := 0 testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { count++ @@ -1508,22 +1509,30 @@ func TestBackoffLifecycle(t *testing.T) { seconds := []int{0, 1, 2, 4, 8, 0, 1, 2, 4, 0} request := c.Verb("POST").Prefix("backofftest").Suffix("abc") clock := testingclock.FakeClock{} - request.backoff = &URLBackoff{ - // Use a fake backoff here to avoid flakes and speed the test up. - Backoff: flowcontrol.NewFakeBackOff( - time.Duration(1)*time.Second, - time.Duration(200)*time.Second, - &clock, - )} + request.backoff = stepClockDuringSleep{ + BackoffManagerWithContext: &URLBackoff{ + // Use a fake backoff here to avoid flakes and speed the test up. + Backoff: flowcontrol.NewFakeBackOff( + time.Duration(1)*time.Second, + time.Duration(200)*time.Second, + &clock, + ), + }, + clock: &clock, + } for _, sec := range seconds { - thisBackoff := request.backoff.CalculateBackoff(request.URL()) + thisBackoff := request.backoff.CalculateBackoffWithContext(ctx, request.URL()) t.Logf("Current backoff %v", thisBackoff) if thisBackoff != time.Duration(sec)*time.Second { t.Errorf("Backoff is %v instead of %v", thisBackoff, sec) } + + // This relies on advancing the fake clock by exactly the duration + // that SleepWithContext is being called for while DoRaw is executing. + // stepClockDuringSleep.SleepWithContext ensures that this happens. now := clock.Now() - request.DoRaw(context.Background()) + request.DoRaw(ctx) elapsed := clock.Since(now) if clock.Since(now) != thisBackoff { t.Errorf("CalculatedBackoff not honored by clock: Expected time of %v, but got %v ", thisBackoff, elapsed) @@ -1531,18 +1540,51 @@ func TestBackoffLifecycle(t *testing.T) { } } +type stepClockDuringSleep struct { + BackoffManagerWithContext + clock *testingclock.FakeClock +} + +// SleepWithContext wraps the underlying SleepWithContext and ensures that once +// that is sleeping, the fake clock advances by exactly the duration that +// it is sleeping for. +func (s stepClockDuringSleep) SleepWithContext(ctx context.Context, d time.Duration) { + // This code is sensitive to both the implementation of + // URLBackoff.SleepWithContext and of FakeClock.NewTimer: + // - SleepWithContext must be a no-op when the duration is zero + // => no need to step the fake clock + // - SleepWithContext must use FakeClock.NewTimer, not FakeClock.Sleep + // because the latter would advance time itself + if d != 0 { + go func() { + // Poll until the caller is sleeping. + for { + if s.clock.HasWaiters() { + s.clock.Step(d) + return + } + if ctx.Err() != nil { + return + } + time.Sleep(time.Millisecond) + } + }() + } + s.BackoffManagerWithContext.SleepWithContext(ctx, d) +} + type testBackoffManager struct { sleeps []time.Duration } -func (b *testBackoffManager) UpdateBackoff(actualUrl *url.URL, err error, responseCode int) { +func (b *testBackoffManager) UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) { } -func (b *testBackoffManager) CalculateBackoff(actualUrl *url.URL) time.Duration { +func (b *testBackoffManager) CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration { return time.Duration(0) } -func (b *testBackoffManager) Sleep(d time.Duration) { +func (b *testBackoffManager) SleepWithContext(ctx context.Context, d time.Duration) { b.sleeps = append(b.sleeps, d) } @@ -1568,7 +1610,7 @@ func TestCheckRetryClosesBody(t *testing.T) { expectedSleeps := []time.Duration{0, time.Second, time.Second, time.Second, time.Second} c := testRESTClient(t, testServer) - c.createBackoffMgr = func() BackoffManager { return backoff } + c.createBackoffMgr = func() BackoffManagerWithContext { return backoff } _, err := c.Verb("POST"). Prefix("foo", "bar"). Suffix("baz"). @@ -2612,6 +2654,8 @@ type noSleepBackOff struct { func (n *noSleepBackOff) Sleep(d time.Duration) {} +func (n *noSleepBackOff) SleepWithContext(ctx context.Context, d time.Duration) {} + func TestRequestWithRetry(t *testing.T) { tests := []struct { name string @@ -2997,7 +3041,6 @@ const retryTestKey retryTestKeyType = iota // metric calls are invoked appropriately in right order. type withRateLimiterBackoffManagerAndMetrics struct { flowcontrol.RateLimiter - *NoBackoff metrics.ResultMetric calculateBackoffSeq int64 calculateBackoffFn func(i int64) time.Duration @@ -3013,7 +3056,7 @@ func (lb *withRateLimiterBackoffManagerAndMetrics) Wait(ctx context.Context) err return nil } -func (lb *withRateLimiterBackoffManagerAndMetrics) CalculateBackoff(actualUrl *url.URL) time.Duration { +func (lb *withRateLimiterBackoffManagerAndMetrics) CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration { lb.invokeOrderGot = append(lb.invokeOrderGot, "BackoffManager.CalculateBackoff") waitFor := lb.calculateBackoffFn(lb.calculateBackoffSeq) @@ -3021,11 +3064,11 @@ func (lb *withRateLimiterBackoffManagerAndMetrics) CalculateBackoff(actualUrl *u return waitFor } -func (lb *withRateLimiterBackoffManagerAndMetrics) UpdateBackoff(actualUrl *url.URL, err error, responseCode int) { +func (lb *withRateLimiterBackoffManagerAndMetrics) UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) { lb.invokeOrderGot = append(lb.invokeOrderGot, "BackoffManager.UpdateBackoff") } -func (lb *withRateLimiterBackoffManagerAndMetrics) Sleep(d time.Duration) { +func (lb *withRateLimiterBackoffManagerAndMetrics) SleepWithContext(ctx context.Context, d time.Duration) { lb.invokeOrderGot = append(lb.invokeOrderGot, "BackoffManager.Sleep") lb.sleepsGot = append(lb.sleepsGot, d.String()) } @@ -3206,7 +3249,6 @@ func testRetryWithRateLimiterBackoffAndMetrics(t *testing.T, key string, doFunc t.Run(test.name, func(t *testing.T) { interceptor := &withRateLimiterBackoffManagerAndMetrics{ RateLimiter: flowcontrol.NewFakeAlwaysRateLimiter(), - NoBackoff: &NoBackoff{}, calculateBackoffFn: test.calculateBackoffFn, } diff --git a/staging/src/k8s.io/client-go/rest/urlbackoff.go b/staging/src/k8s.io/client-go/rest/urlbackoff.go index 2f9962d7e54..5b7b4e216ef 100644 --- a/staging/src/k8s.io/client-go/rest/urlbackoff.go +++ b/staging/src/k8s.io/client-go/rest/urlbackoff.go @@ -17,6 +17,8 @@ limitations under the License. package rest import ( + "context" + "fmt" "net/url" "time" @@ -32,12 +34,24 @@ import ( var serverIsOverloadedSet = sets.NewInt(429) var maxResponseCode = 499 +//go:generate mockery + +// Deprecated: BackoffManager.Sleep ignores the caller's context. Use BackoffManagerWithContext instead. type BackoffManager interface { - UpdateBackoff(actualUrl *url.URL, err error, responseCode int) - CalculateBackoff(actualUrl *url.URL) time.Duration + UpdateBackoff(actualURL *url.URL, err error, responseCode int) + CalculateBackoff(actualURL *url.URL) time.Duration Sleep(d time.Duration) } +type BackoffManagerWithContext interface { + UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) + CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration + SleepWithContext(ctx context.Context, d time.Duration) +} + +var _ BackoffManager = &URLBackoff{} +var _ BackoffManagerWithContext = &URLBackoff{} + // URLBackoff struct implements the semantics on top of Backoff which // we need for URL specific exponential backoff. type URLBackoff struct { @@ -49,11 +63,19 @@ type URLBackoff struct { type NoBackoff struct { } -func (n *NoBackoff) UpdateBackoff(actualUrl *url.URL, err error, responseCode int) { +func (n *NoBackoff) UpdateBackoff(actualURL *url.URL, err error, responseCode int) { // do nothing. } -func (n *NoBackoff) CalculateBackoff(actualUrl *url.URL) time.Duration { +func (n *NoBackoff) UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) { + // do nothing. +} + +func (n *NoBackoff) CalculateBackoff(actualURL *url.URL) time.Duration { + return 0 * time.Second +} + +func (n *NoBackoff) CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration { return 0 * time.Second } @@ -61,10 +83,21 @@ func (n *NoBackoff) Sleep(d time.Duration) { time.Sleep(d) } +func (n *NoBackoff) SleepWithContext(ctx context.Context, d time.Duration) { + if d == 0 { + return + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + case <-t.C: + } +} + // Disable makes the backoff trivial, i.e., sets it to zero. This might be used // by tests which want to run 1000s of mock requests without slowing down. func (b *URLBackoff) Disable() { - klog.V(4).Infof("Disabling backoff strategy") b.Backoff = flowcontrol.NewBackOff(0*time.Second, 0*time.Second) } @@ -76,32 +109,74 @@ func (b *URLBackoff) baseUrlKey(rawurl *url.URL) string { // in the future. host, err := url.Parse(rawurl.String()) if err != nil { - klog.V(4).Infof("Error extracting url: %v", rawurl) - panic("bad url!") + panic(fmt.Sprintf("Error parsing bad URL %q: %v", rawurl, err)) } return host.Host } // UpdateBackoff updates backoff metadata -func (b *URLBackoff) UpdateBackoff(actualUrl *url.URL, err error, responseCode int) { +func (b *URLBackoff) UpdateBackoff(actualURL *url.URL, err error, responseCode int) { + b.UpdateBackoffWithContext(context.Background(), actualURL, err, responseCode) +} + +// UpdateBackoffWithContext updates backoff metadata +func (b *URLBackoff) UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) { // range for retry counts that we store is [0,13] if responseCode > maxResponseCode || serverIsOverloadedSet.Has(responseCode) { - b.Backoff.Next(b.baseUrlKey(actualUrl), b.Backoff.Clock.Now()) + b.Backoff.Next(b.baseUrlKey(actualURL), b.Backoff.Clock.Now()) return } else if responseCode >= 300 || err != nil { - klog.V(4).Infof("Client is returning errors: code %v, error %v", responseCode, err) + klog.FromContext(ctx).V(4).Info("Client is returning errors", "code", responseCode, "err", err) } //If we got this far, there is no backoff required for this URL anymore. - b.Backoff.Reset(b.baseUrlKey(actualUrl)) + b.Backoff.Reset(b.baseUrlKey(actualURL)) } // CalculateBackoff takes a url and back's off exponentially, // based on its knowledge of existing failures. -func (b *URLBackoff) CalculateBackoff(actualUrl *url.URL) time.Duration { - return b.Backoff.Get(b.baseUrlKey(actualUrl)) +func (b *URLBackoff) CalculateBackoff(actualURL *url.URL) time.Duration { + return b.Backoff.Get(b.baseUrlKey(actualURL)) +} + +// CalculateBackoffWithContext takes a url and back's off exponentially, +// based on its knowledge of existing failures. +func (b *URLBackoff) CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration { + return b.Backoff.Get(b.baseUrlKey(actualURL)) } func (b *URLBackoff) Sleep(d time.Duration) { b.Backoff.Clock.Sleep(d) } + +func (b *URLBackoff) SleepWithContext(ctx context.Context, d time.Duration) { + if d == 0 { + return + } + t := b.Backoff.Clock.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + case <-t.C(): + } +} + +// backoffManagerNopContext wraps a BackoffManager and adds the *WithContext methods. +type backoffManagerNopContext struct { + BackoffManager +} + +var _ BackoffManager = &backoffManagerNopContext{} +var _ BackoffManagerWithContext = &backoffManagerNopContext{} + +func (b *backoffManagerNopContext) UpdateBackoffWithContext(ctx context.Context, actualURL *url.URL, err error, responseCode int) { + b.UpdateBackoff(actualURL, err, responseCode) +} + +func (b *backoffManagerNopContext) CalculateBackoffWithContext(ctx context.Context, actualURL *url.URL) time.Duration { + return b.CalculateBackoff(actualURL) +} + +func (b *backoffManagerNopContext) SleepWithContext(ctx context.Context, d time.Duration) { + b.Sleep(d) +} diff --git a/staging/src/k8s.io/client-go/rest/urlbackoff_test.go b/staging/src/k8s.io/client-go/rest/urlbackoff_test.go index c5f439238d6..b80b721d974 100644 --- a/staging/src/k8s.io/client-go/rest/urlbackoff_test.go +++ b/staging/src/k8s.io/client-go/rest/urlbackoff_test.go @@ -17,10 +17,14 @@ limitations under the License. package rest import ( + "context" + "errors" "net/url" "testing" "time" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" ) @@ -77,3 +81,38 @@ func TestURLBackoffFunctionality(t *testing.T) { t.Errorf("The final return code %v should have resulted in a backoff ! ", returnCodes[7]) } } + +func TestBackoffManagerNopContext(t *testing.T) { + mock := NewMockBackoffManager(t) + + sleepDuration := 42 * time.Second + mock.On("Sleep", sleepDuration).Return() + url := &url.URL{} + mock.On("CalculateBackoff", url).Return(time.Second) + err := errors.New("fake error") + responseCode := 404 + mock.On("UpdateBackoff", url, err, responseCode).Return() + + ctx := context.Background() + wrapper := backoffManagerNopContext{BackoffManager: mock} + wrapper.SleepWithContext(ctx, sleepDuration) + wrapper.CalculateBackoffWithContext(ctx, url) + wrapper.UpdateBackoffWithContext(ctx, url, err, responseCode) +} + +func TestNoBackoff(t *testing.T) { + var backoff NoBackoff + assert.Equal(t, 0*time.Second, backoff.CalculateBackoff(nil)) + assert.Equal(t, 0*time.Second, backoff.CalculateBackoffWithContext(context.Background(), nil)) + + start := time.Now() + backoff.Sleep(0 * time.Second) + assert.WithinDuration(t, start, time.Now(), time.Minute /* pretty generous, but we don't want to flake */, time.Since(start), "backoff.Sleep") + + // Cancel right away to prevent sleeping. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start = time.Now() + backoff.SleepWithContext(ctx, 10*time.Minute) + assert.WithinDuration(t, start, time.Now(), time.Minute /* pretty generous, but we don't want to flake */, time.Since(start), "backoff.SleepWithContext") +} diff --git a/staging/src/k8s.io/client-go/rest/with_retry.go b/staging/src/k8s.io/client-go/rest/with_retry.go index eaaadc6a4c3..eb7eaaf37d9 100644 --- a/staging/src/k8s.io/client-go/rest/with_retry.go +++ b/staging/src/k8s.io/client-go/rest/with_retry.go @@ -209,18 +209,18 @@ func (r *withRetry) Before(ctx context.Context, request *Request) error { // we do a backoff sleep before the first attempt is made, // (preserving current behavior). if request.backoff != nil { - request.backoff.Sleep(request.backoff.CalculateBackoff(url)) + request.backoff.SleepWithContext(ctx, request.backoff.CalculateBackoffWithContext(ctx, url)) } return nil } // if we are here, we have made attempt(s) at least once before. if request.backoff != nil { - delay := request.backoff.CalculateBackoff(url) + delay := request.backoff.CalculateBackoffWithContext(ctx, url) if r.retryAfter.Wait > delay { delay = r.retryAfter.Wait } - request.backoff.Sleep(delay) + request.backoff.SleepWithContext(ctx, delay) } // We are retrying the request that we already send to @@ -258,9 +258,9 @@ func (r *withRetry) After(ctx context.Context, request *Request, resp *http.Resp if request.c.base != nil { if err != nil { - request.backoff.UpdateBackoff(request.URL(), err, 0) + request.backoff.UpdateBackoffWithContext(ctx, request.URL(), err, 0) } else { - request.backoff.UpdateBackoff(request.URL(), err, resp.StatusCode) + request.backoff.UpdateBackoffWithContext(ctx, request.URL(), err, resp.StatusCode) } } } diff --git a/staging/src/k8s.io/cloud-provider/go.sum b/staging/src/k8s.io/cloud-provider/go.sum index 2e70ad0f92f..1498fee5d67 100644 --- a/staging/src/k8s.io/cloud-provider/go.sum +++ b/staging/src/k8s.io/cloud-provider/go.sum @@ -168,6 +168,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/code-generator/examples/go.sum b/staging/src/k8s.io/code-generator/examples/go.sum index bd864cd94ca..ee5719676fd 100644 --- a/staging/src/k8s.io/code-generator/examples/go.sum +++ b/staging/src/k8s.io/code-generator/examples/go.sum @@ -70,6 +70,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= diff --git a/staging/src/k8s.io/component-base/go.sum b/staging/src/k8s.io/component-base/go.sum index c5a1ad455b8..4a465041b36 100644 --- a/staging/src/k8s.io/component-base/go.sum +++ b/staging/src/k8s.io/component-base/go.sum @@ -131,6 +131,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/component-helpers/go.sum b/staging/src/k8s.io/component-helpers/go.sum index 4742813c291..aad3b05cba2 100644 --- a/staging/src/k8s.io/component-helpers/go.sum +++ b/staging/src/k8s.io/component-helpers/go.sum @@ -79,6 +79,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/controller-manager/go.sum b/staging/src/k8s.io/controller-manager/go.sum index 200be8986ab..c595c9d578d 100644 --- a/staging/src/k8s.io/controller-manager/go.sum +++ b/staging/src/k8s.io/controller-manager/go.sum @@ -164,6 +164,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/dynamic-resource-allocation/go.sum b/staging/src/k8s.io/dynamic-resource-allocation/go.sum index 219f6c5f016..fa6514375ca 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/go.sum +++ b/staging/src/k8s.io/dynamic-resource-allocation/go.sum @@ -139,6 +139,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/endpointslice/go.sum b/staging/src/k8s.io/endpointslice/go.sum index 4e8ebff68cd..ff71afe8bc6 100644 --- a/staging/src/k8s.io/endpointslice/go.sum +++ b/staging/src/k8s.io/endpointslice/go.sum @@ -109,6 +109,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index e7d8e4b419b..5fdad9a6098 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -167,6 +167,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/kubelet/go.sum b/staging/src/k8s.io/kubelet/go.sum index 3705cbf6f4d..2b5206912ec 100644 --- a/staging/src/k8s.io/kubelet/go.sum +++ b/staging/src/k8s.io/kubelet/go.sum @@ -131,6 +131,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/staging/src/k8s.io/metrics/go.sum b/staging/src/k8s.io/metrics/go.sum index 7bbf85211b5..ef906d0fd7c 100644 --- a/staging/src/k8s.io/metrics/go.sum +++ b/staging/src/k8s.io/metrics/go.sum @@ -79,6 +79,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/pod-security-admission/go.sum b/staging/src/k8s.io/pod-security-admission/go.sum index 200be8986ab..c595c9d578d 100644 --- a/staging/src/k8s.io/pod-security-admission/go.sum +++ b/staging/src/k8s.io/pod-security-admission/go.sum @@ -164,6 +164,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index c8c7b6f1e40..cde9f6c8c26 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -164,6 +164,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/sample-controller/go.sum b/staging/src/k8s.io/sample-controller/go.sum index 85879c8d8a2..af1e7d7e0cd 100644 --- a/staging/src/k8s.io/sample-controller/go.sum +++ b/staging/src/k8s.io/sample-controller/go.sum @@ -79,6 +79,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=