Merge pull request #51876 from smarterclayton/disable_client_paging

Automatic merge from submit-queue (batch tested with PRs 48552, 51876)

Disable default paging in list watches

For 1.8 this will be off by default. In 1.9 it will be on by default.
Add tests and rename some fields to use the `chunking` terminology.

Note that the pager may be used for other things besides chunking.

Follow on to #48921, we left the field on to get some exercise in the normal code paths, but needs to be disabled for 1.8.

@liggitt let's merge on wednesday.

Kubernetes-commit: eda3db550bd72499d735938a1911f9f7e88870f8
This commit is contained in:
Kubernetes Publisher 2017-09-07 23:08:17 -07:00
commit fa6f560e89
5 changed files with 284 additions and 55 deletions

98
Godeps/Godeps.json generated
View File

@ -472,199 +472,199 @@
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apimachinery",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1alpha1",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion/unstructured",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/fields",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/labels",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/selection",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/types",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream/spdy",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/json",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/net",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/remotecommand",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/version",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/watch",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/netutil",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
"Rev": "f02a00c7b3a541adf8a17951f68899f25c410066"
"Rev": "a28c9b0b532638dcfb2c8b1b6f8f1e4a21041752"
},
{
"ImportPath": "k8s.io/kube-openapi/pkg/common",

View File

@ -49,9 +49,11 @@ type WatchFunc func(options metav1.ListOptions) (watch.Interface, error)
// It is a convenience function for users of NewReflector, etc.
// ListFunc and WatchFunc must not be nil
type ListWatch struct {
ListFunc ListFunc
WatchFunc WatchFunc
DisablePaging bool
ListFunc ListFunc
WatchFunc WatchFunc
// DisableChunking requests no chunking for this list watcher. It has no effect in Kubernetes 1.8, but in
// 1.9 will allow a controller to opt out of chunking.
DisableChunking bool
}
// Getter interface knows how to access Get method from RESTClient.
@ -91,7 +93,9 @@ func timeoutFromListOptions(options metav1.ListOptions) time.Duration {
// List a set of apiserver resources
func (lw *ListWatch) List(options metav1.ListOptions) (runtime.Object, error) {
if !lw.DisablePaging {
// chunking will become the default for list watchers starting in Kubernetes 1.9, unless
// otherwise disabled.
if false && !lw.DisableChunking {
return pager.New(pager.SimplePageFunc(lw.ListFunc)).List(context.TODO(), options)
}
return lw.ListFunc(options)

View File

@ -5,6 +5,7 @@ licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
@ -34,3 +35,17 @@ filegroup(
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["pager_test.go"],
library = ":go_default_library",
deps = [
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/internalversion:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1alpha1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
],
)

View File

@ -52,7 +52,8 @@ type ListPager struct {
}
// New creates a new pager from the provided pager function using the default
// options.
// options. It will fall back to a full list if an expiration error is encountered
// as a last resort.
func New(fn ListPageFunc) *ListPager {
return &ListPager{
PageSize: defaultPageSize,
@ -61,9 +62,12 @@ func New(fn ListPageFunc) *ListPager {
}
}
// TODO: introduce other types of paging functions - such as those that retrieve from a list
// of namespaces.
// List returns a single list object, but attempts to retrieve smaller chunks from the
// server to reduce the impact on the server. If the chunk attempt fails, it will load
// the full list instead.
// the full list instead. The Limit field on options, if unset, will default to the page size.
func (p *ListPager) List(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
if options.Limit == 0 {
options.Limit = p.PageSize

206
tools/pager/pager_test.go Normal file
View File

@ -0,0 +1,206 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pager
import (
"fmt"
"reflect"
"testing"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/api/errors"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1alpha1 "k8s.io/apimachinery/pkg/apis/meta/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
)
func list(count int, rv string) *metainternalversion.List {
var list metainternalversion.List
for i := 0; i < count; i++ {
list.Items = append(list.Items, &metav1alpha1.PartialObjectMetadata{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%d", i),
},
})
}
list.ResourceVersion = rv
return &list
}
type testPager struct {
t *testing.T
rv string
index int
remaining int
last int
continuing bool
done bool
expectPage int64
}
func (p *testPager) reset() {
p.continuing = false
p.remaining += p.index
p.index = 0
p.last = 0
p.done = false
}
func (p *testPager) PagedList(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
if p.done {
p.t.Errorf("did not expect additional call to paged list")
return nil, fmt.Errorf("unexpected list call")
}
expectedContinue := fmt.Sprintf("%s:%d", p.rv, p.last)
if options.Limit != p.expectPage || (p.continuing && options.Continue != expectedContinue) {
p.t.Errorf("invariant violated, expected limit %d and continue %s, got %#v", p.expectPage, expectedContinue, options)
return nil, fmt.Errorf("invariant violated")
}
var list metainternalversion.List
total := options.Limit
if total == 0 {
total = int64(p.remaining)
}
for i := int64(0); i < total; i++ {
if p.remaining <= 0 {
break
}
list.Items = append(list.Items, &metav1alpha1.PartialObjectMetadata{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%d", p.index),
},
})
p.remaining--
p.index++
}
p.last = p.index
if p.remaining > 0 {
list.Continue = fmt.Sprintf("%s:%d", p.rv, p.last)
p.continuing = true
} else {
p.done = true
}
list.ResourceVersion = p.rv
return &list, nil
}
func (p *testPager) ExpiresOnSecondPage(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
if p.continuing {
p.done = true
return nil, errors.NewResourceExpired("this list has expired")
}
return p.PagedList(ctx, options)
}
func (p *testPager) ExpiresOnSecondPageThenFullList(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
if p.continuing {
p.reset()
p.expectPage = 0
return nil, errors.NewResourceExpired("this list has expired")
}
return p.PagedList(ctx, options)
}
func TestListPager_List(t *testing.T) {
type fields struct {
PageSize int64
PageFn ListPageFunc
FullListIfExpired bool
}
type args struct {
ctx context.Context
options metav1.ListOptions
}
tests := []struct {
name string
fields fields
args args
want runtime.Object
wantErr bool
isExpired bool
}{
{
name: "empty page",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 0, rv: "rv:20"}).PagedList},
args: args{},
want: list(0, "rv:20"),
},
{
name: "one page",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 9, rv: "rv:20"}).PagedList},
args: args{},
want: list(9, "rv:20"),
},
{
name: "one full page",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 10, rv: "rv:20"}).PagedList},
args: args{},
want: list(10, "rv:20"),
},
{
name: "two pages",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 11, rv: "rv:20"}).PagedList},
args: args{},
want: list(11, "rv:20"),
},
{
name: "three pages",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 21, rv: "rv:20"}).PagedList},
args: args{},
want: list(21, "rv:20"),
},
{
name: "expires on second page",
fields: fields{PageSize: 10, PageFn: (&testPager{t: t, expectPage: 10, remaining: 21, rv: "rv:20"}).ExpiresOnSecondPage},
args: args{},
wantErr: true,
isExpired: true,
},
{
name: "expires on second page and then lists",
fields: fields{
FullListIfExpired: true,
PageSize: 10,
PageFn: (&testPager{t: t, expectPage: 10, remaining: 21, rv: "rv:20"}).ExpiresOnSecondPageThenFullList,
},
args: args{},
want: list(21, "rv:20"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &ListPager{
PageSize: tt.fields.PageSize,
PageFn: tt.fields.PageFn,
FullListIfExpired: tt.fields.FullListIfExpired,
}
got, err := p.List(tt.args.ctx, tt.args.options)
if (err != nil) != tt.wantErr {
t.Errorf("ListPager.List() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.isExpired != errors.IsResourceExpired(err) {
t.Errorf("ListPager.List() error = %v, isExpired %v", err, tt.isExpired)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ListPager.List() = %v, want %v", got, tt.want)
}
})
}
}