Compare commits

..

9 Commits

Author SHA1 Message Date
Kubernetes Publisher
c76a06cc7d Update dependencies to v0.22.6 tag 2022-01-19 21:42:35 +00:00
Kubernetes Publisher
74d3410324 Merge pull request #107568 from jiahuif-forks/automated-cherry-pick-of-#107565-upstream-release-1.22
Automated cherry pick of #107565: upgrade sigs.k8s.io/structured-merge-diff/v4 to v4.2.1

Kubernetes-commit: 487eef699dc8ad07106876e7fb045962606035ec
2022-01-15 13:27:08 +00:00
Kubernetes Publisher
558b220907 Merge pull request #106582 from hzxuzhonghu/automated-cherry-pick-of-#104991-#105031-origin-release-1.22
Automated cherry pick of #104991: Fix workqueue memory leak
#105031: workqueue: fix leak in queue preventing objects from being

Kubernetes-commit: dcb59ac5e2ae03a5284398248fe5162d5814ef67
2022-01-15 05:48:19 +00:00
Jiahui Feng
920722c69c generated: ./hack/update-vendor.sh
Kubernetes-commit: e0550f5f956b7fac6033bb58c678f733d972c97d
2022-01-14 10:31:44 -08:00
Jiahui Feng
a33ed41d67 upgrade sigs.k8s.io/structured-merge-diff/v4 to v4.2.1
Kubernetes-commit: fa2db4831f720fcc31139a026cb012130058bca3
2022-01-14 10:30:23 -08:00
Kubernetes Publisher
81c3757ed9 Merge pull request #107335 from fasaxc/automated-cherry-pick-of-#107311-upstream-release-1.22
Automated cherry pick of #107311: client-go: Clear the ResourceVersionMatch on paged list calls

Kubernetes-commit: 7ae06380be3782c228a4f6ca44d5af5975f286c5
2022-01-05 17:38:27 -08:00
Shaun Crampton
584bf4e4e8 client-go: Clear the ResourceVersionMatch on paged list calls
API server rejects continuations with ResourceVersionMatch set.

Kubernetes-commit: a1ab5cbbf147aab16d087adb5b2ff58239fa67d1
2022-01-04 16:05:32 +00:00
John Howard
973eb806bc workqueue: fix leak in queue preventing objects from being GCed
See https://github.com/grpc/grpc-go/issues/4758 for a real world example
of this leaking 2gb+ of data.

Basically, when we do `q.queue[1:]` we are just repositioning the slice.
The underlying array is still active, which contains the object formerly
known as `q.queue[0]`. Because its referencing this object, it will not
be GCed. The only thing that will trigger it to free is eventually when
we add enough to the queue that we allocate a whole new array.

Instead, we should explicitly clear out the old space when we remove it
from the queue. This ensures the object can be GCed, assuming the users'
application doesn't reference it anymore.

Kubernetes-commit: fe4e0cb3f518144b8892414992350783f2f20c07
2021-09-14 15:51:09 -07:00
xuzhonghu
7f47bbe3fe Fix workqueue memory leak
Kubernetes-commit: ccc8303df1970632e9ce8576f24ca5b20115f5fa
2021-09-14 09:53:53 +08:00
6 changed files with 70 additions and 13 deletions

10
go.mod
View File

@@ -30,15 +30,15 @@ require (
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
google.golang.org/protobuf v1.26.0
k8s.io/api v0.22.5
k8s.io/apimachinery v0.22.5
k8s.io/api v0.22.6
k8s.io/apimachinery v0.22.6
k8s.io/klog/v2 v2.9.0
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
sigs.k8s.io/structured-merge-diff/v4 v4.1.2
sigs.k8s.io/structured-merge-diff/v4 v4.2.1
sigs.k8s.io/yaml v1.2.0
)
replace (
k8s.io/api => k8s.io/api v0.22.5
k8s.io/apimachinery => k8s.io/apimachinery v0.22.5
k8s.io/api => k8s.io/api v0.22.6
k8s.io/apimachinery => k8s.io/apimachinery v0.22.6
)

12
go.sum
View File

@@ -444,10 +444,10 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.22.5 h1:xk7C+rMjF/EGELiD560jdmwzrB788mfcHiNbMQLIVI8=
k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s=
k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
k8s.io/api v0.22.6 h1:acjE5ABt0KpsBI9QCtLqaQEPSF94jOtE/LoFxSYasSE=
k8s.io/api v0.22.6/go.mod h1:q1F7IfaNrbi/83ebLy3YFQYLjPSNyunZ/IXQxMmbwCg=
k8s.io/apimachinery v0.22.6 h1:z7vxNRkFX0NToA+8D17kzLZ/T4t+DqwzUlqqbqRepRs=
k8s.io/apimachinery v0.22.6/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
@@ -460,7 +460,7 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

View File

@@ -78,6 +78,7 @@ func (p *ListPager) List(ctx context.Context, options metav1.ListOptions) (runti
options.Limit = p.PageSize
}
requestedResourceVersion := options.ResourceVersion
requestedResourceVersionMatch := options.ResourceVersionMatch
var list *metainternalversion.List
paginatedResult := false
@@ -102,6 +103,7 @@ func (p *ListPager) List(ctx context.Context, options metav1.ListOptions) (runti
options.Limit = 0
options.Continue = ""
options.ResourceVersion = requestedResourceVersion
options.ResourceVersionMatch = requestedResourceVersionMatch
result, err := p.PageFn(ctx, options)
return result, paginatedResult, err
}
@@ -135,10 +137,11 @@ func (p *ListPager) List(ctx context.Context, options metav1.ListOptions) (runti
// set the next loop up
options.Continue = m.GetContinue()
// Clear the ResourceVersion on the subsequent List calls to avoid the
// Clear the ResourceVersion(Match) on the subsequent List calls to avoid the
// `specifying resource version is not allowed when using continue` error.
// See https://github.com/kubernetes/kubernetes/issues/85221#issuecomment-553748143.
options.ResourceVersion = ""
options.ResourceVersionMatch = ""
// At this point, result is already paginated.
paginatedResult = true
}

View File

@@ -76,6 +76,10 @@ func (p *testPager) PagedList(ctx context.Context, options metav1.ListOptions) (
p.t.Errorf("invariant violated, specifying resource version (%s) is not allowed when using continue (%s).", options.ResourceVersion, options.Continue)
return nil, fmt.Errorf("invariant violated")
}
if options.Continue != "" && options.ResourceVersionMatch != "" {
p.t.Errorf("invariant violated, specifying resource version match type (%s) is not allowed when using continue (%s).", options.ResourceVersionMatch, options.Continue)
return nil, fmt.Errorf("invariant violated")
}
var list metainternalversion.List
total := options.Limit
if total == 0 {
@@ -201,6 +205,13 @@ func TestListPager_List(t *testing.T) {
want: list(11, "rv:20"),
wantPaged: true,
},
{
name: "two pages with resourceVersion and resourceVersionMatch",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 11, rv: "rv:20"}).PagedList},
args: args{options: metav1.ListOptions{ResourceVersion: "rv:10", ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan}},
want: list(11, "rv:20"),
wantPaged: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View File

@@ -155,7 +155,10 @@ func (q *Type) Get() (item interface{}, shutdown bool) {
return nil, true
}
item, q.queue = q.queue[0], q.queue[1:]
item = q.queue[0]
// The underlying array still exists and reference this object, so the object will not be garbage collected.
q.queue[0] = nil
q.queue = q.queue[1:]
q.metrics.get(item)

View File

@@ -17,10 +17,13 @@ limitations under the License.
package workqueue_test
import (
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
)
@@ -159,3 +162,40 @@ func TestReinsert(t *testing.T) {
t.Errorf("Expected queue to be empty. Has %v items", a)
}
}
// TestGarbageCollection ensures that objects that are added then removed from the queue are
// able to be garbage collected.
func TestGarbageCollection(t *testing.T) {
type bigObject struct {
data []byte
}
leakQueue := workqueue.New()
t.Cleanup(func() {
// Make sure leakQueue doesn't go out of scope too early
runtime.KeepAlive(leakQueue)
})
c := &bigObject{data: []byte("hello")}
mustGarbageCollect(t, c)
leakQueue.Add(c)
o, _ := leakQueue.Get()
leakQueue.Done(o)
}
// mustGarbageCollect asserts than an object was garbage collected by the end of the test.
// The input must be a pointer to an object.
func mustGarbageCollect(t *testing.T, i interface{}) {
t.Helper()
var collected int32 = 0
runtime.SetFinalizer(i, func(x interface{}) {
atomic.StoreInt32(&collected, 1)
})
t.Cleanup(func() {
if err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (done bool, err error) {
// Trigger GC explicitly, otherwise we may need to wait a long time for it to run
runtime.GC()
return atomic.LoadInt32(&collected) == 1, nil
}); err != nil {
t.Errorf("object was not garbage collected")
}
})
}