diff --git a/go.mod b/go.mod index 884a57c..52b7f06 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 - github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076 + github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd github.com/rancher/dynamiclistener v0.3.5 github.com/rancher/kubernetes-provider-detector v0.1.2 github.com/rancher/norman v0.0.0-20221205184727-32ef2e185b99 diff --git a/go.sum b/go.sum index f321ea8..6c0994d 100644 --- a/go.sum +++ b/go.sum @@ -502,8 +502,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076 h1:wS95KbXFI1QOVQr3Tz+qyOJ9iia1ITCnjsapxJyI/9U= -github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA= +github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd h1:g0hNrbONfmY4lxvrD2q9KkueYYY4wKUYscm6Ih0QfQ0= +github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA= github.com/rancher/client-go v1.25.4-rancher1 h1:9MlBC8QbgngUkhNzMR8rZmmCIj6WNRHFOnYiwC2Kty4= github.com/rancher/client-go v1.25.4-rancher1/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A= diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index c311d1d..172312c 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -18,14 +18,17 @@ const ( limitParam = "limit" filterParam = "filter" sortParam = "sort" + pageSizeParam = "pagesize" + pageParam = "page" ) // ListOptions represents the query parameters that may be included in a list request. type ListOptions struct { - ChunkSize int - Resume string - Filters []Filter - Sort Sort + ChunkSize int + Resume string + Filters []Filter + Sort Sort + Pagination Pagination } // Filter represents a field to filter by. @@ -55,6 +58,12 @@ type Sort struct { order SortOrder } +// Pagination represents how to return paginated results. +type Pagination struct { + pageSize int + page int +} + // ParseQuery parses the query params of a request and returns a ListOptions. func ParseQuery(apiOp *types.APIRequest) *ListOptions { chunkSize := getLimit(apiOp) @@ -78,11 +87,22 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { if sortKey != "" { sort.field = strings.Split(sortKey, ".") } + var err error + pagination := Pagination{} + pagination.pageSize, err = strconv.Atoi(q.Get(pageSizeParam)) + if err != nil { + pagination.pageSize = 0 + } + pagination.page, err = strconv.Atoi(q.Get(pageParam)) + if err != nil { + pagination.page = 1 + } return &ListOptions{ - ChunkSize: chunkSize, - Resume: cont, - Filters: filterOpts, - Sort: sort, + ChunkSize: chunkSize, + Resume: cont, + Filters: filterOpts, + Sort: sort, + Pagination: pagination, } } @@ -196,3 +216,26 @@ func SortList(list []unstructured.Unstructured, s Sort) []unstructured.Unstructu }) return list } + +// PaginateList returns a subset of the result based on the pagination criteria as well as the total number of pages the caller can expect. +func PaginateList(list []unstructured.Unstructured, p Pagination) ([]unstructured.Unstructured, int) { + if p.pageSize <= 0 { + return list, 0 + } + page := p.page - 1 + if p.page < 1 { + page = 0 + } + pages := len(list) / p.pageSize + if len(list)%p.pageSize != 0 { + pages++ + } + offset := p.pageSize * page + if offset > len(list) { + return []unstructured.Unstructured{}, pages + } + if offset+p.pageSize > len(list) { + return list[offset:], pages + } + return list[offset : offset+p.pageSize], pages +} diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go index b418ff6..4e867fc 100644 --- a/pkg/stores/partition/listprocessor/processor_test.go +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -1115,3 +1115,202 @@ func TestSortList(t *testing.T) { }) } } + +func TestPaginateList(t *testing.T) { + objects := []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "red-delicious", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "crispin", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "bramley", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "golden-delicious", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "macintosh", + }, + }, + }, + } + tests := []struct { + name string + objects []unstructured.Unstructured + pagination Pagination + want []unstructured.Unstructured + wantPages int + }{ + { + name: "pagesize=3, page=unset", + objects: objects, + pagination: Pagination{ + pageSize: 3, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=3, page=1", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 1, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=3, page=2", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 2, + }, + want: objects[3:6], + wantPages: 3, + }, + { + name: "pagesize=3, page=last", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 3, + }, + want: objects[6:], + wantPages: 3, + }, + { + name: "pagesize=3, page>last", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 37, + }, + want: []unstructured.Unstructured{}, + wantPages: 3, + }, + { + name: "pagesize=3, page<0", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: -4, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=0", + objects: objects, + pagination: Pagination{}, + want: objects, + wantPages: 0, + }, + { + name: "pagesize=-1", + objects: objects, + pagination: Pagination{ + pageSize: -7, + }, + want: objects, + wantPages: 0, + }, + { + name: "even page size, even list size", + objects: objects, + pagination: Pagination{ + pageSize: 2, + page: 2, + }, + want: objects[2:4], + wantPages: 4, + }, + { + name: "even page size, odd list size", + objects: objects[1:], + pagination: Pagination{ + pageSize: 2, + page: 2, + }, + want: objects[3:5], + wantPages: 4, + }, + { + name: "odd page size, even list size", + objects: objects, + pagination: Pagination{ + pageSize: 5, + page: 2, + }, + want: objects[5:], + wantPages: 2, + }, + { + name: "odd page size, odd list size", + objects: objects[1:], + pagination: Pagination{ + pageSize: 3, + page: 2, + }, + want: objects[4:7], + wantPages: 3, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, gotPages := PaginateList(test.objects, test.pagination) + assert.Equal(t, test.want, got) + assert.Equal(t, test.wantPages, gotPages) + }) + } +} diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index 0b3dcc0..6d256f8 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -129,6 +129,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP list := listprocessor.FilterList(stream, opts.Filters) list = listprocessor.SortList(list, opts.Sort) + result.Count = len(list) + list, pages := listprocessor.PaginateList(list, opts.Pagination) for _, item := range list { item := item @@ -137,6 +139,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP result.Revision = lister.Revision() result.Continue = lister.Continue() + result.Pages = pages + return result, lister.Err() } diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index fd65187..4b50c25 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -42,6 +42,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 1, Objects: []types.APIObject{ newApple("fuji").toObj(), }, @@ -71,18 +72,21 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 1, Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))), Objects: []types.APIObject{ newApple("fuji").toObj(), }, }, { + Count: 1, Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))), Objects: []types.APIObject{ newApple("granny-smith").toObj(), }, }, { + Count: 1, Objects: []types.APIObject{ newApple("crispin").toObj(), }, @@ -121,6 +125,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("crispin").toObj(), @@ -176,6 +181,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 3, Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)), Objects: []types.APIObject{ newApple("fuji").toObj(), @@ -184,6 +190,7 @@ func TestList(t *testing.T) { }, }, { + Count: 3, Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)), Objects: []types.APIObject{ newApple("bramley").toObj(), @@ -192,6 +199,7 @@ func TestList(t *testing.T) { }, }, { + Count: 1, Objects: []types.APIObject{ newApple("red-delicious").toObj(), }, @@ -221,12 +229,14 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("bramley").toObj(), }, }, { + Count: 1, Objects: []types.APIObject{ newApple("bramley").toObj(), }, @@ -270,6 +280,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 3, Objects: []types.APIObject{ newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(), newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(), @@ -301,6 +312,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 4, Objects: []types.APIObject{ newApple("bramley").toObj(), newApple("crispin").toObj(), @@ -309,6 +321,7 @@ func TestList(t *testing.T) { }, }, { + Count: 4, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("fuji").toObj(), @@ -350,6 +363,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("crispin").toObj(), newApple("granny-smith").toObj(),