Add pagination to partition store

Extend the partition store to parse page and pagesize parameters and
return a subset of list results. The total number of pages is included
in the list response.

Request an initial page:

GET /v1/secrets?pagesize=10

or

GET /v1/secrets?pagesize=10&page=1

Request subsequent pages, or arbitrary pages:

GET /v1/secrets?pagesize=10&page=37

If a page number is out of bounds, an empty list is returned.

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
-	github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076
+	github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd
 	github.com/rancher/dynamiclistener v0.3.5
 	github.com/rancher/kubernetes-provider-detector v0.1.2
 	github.com/rancher/norman v0.0.0-20221205184727-32ef2e185b99

go.sum

@@ -502,8 +502,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076 h1:wS95KbXFI1QOVQr3Tz+qyOJ9iia1ITCnjsapxJyI/9U=
-github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA=
+github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd h1:g0hNrbONfmY4lxvrD2q9KkueYYY4wKUYscm6Ih0QfQ0=
+github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA=
 github.com/rancher/client-go v1.25.4-rancher1 h1:9MlBC8QbgngUkhNzMR8rZmmCIj6WNRHFOnYiwC2Kty4=
 github.com/rancher/client-go v1.25.4-rancher1/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
 github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A=

pkg/stores/partition/listprocessor/processor.go

@@ -18,14 +18,17 @@ const (
 	limitParam    = "limit"
 	filterParam   = "filter"
 	sortParam     = "sort"
+	pageSizeParam = "pagesize"
+	pageParam     = "page"
 )
 
 // ListOptions represents the query parameters that may be included in a list request.
 type ListOptions struct {
-	ChunkSize int
-	Resume    string
-	Filters   []Filter
-	Sort      Sort
+	ChunkSize  int
+	Resume     string
+	Filters    []Filter
+	Sort       Sort
+	Pagination Pagination
 }
 
 // Filter represents a field to filter by.
@@ -55,6 +58,12 @@ type Sort struct {
 	order SortOrder
 }
 
+// Pagination represents how to return paginated results.
+type Pagination struct {
+	pageSize int
+	page     int
+}
+
 // ParseQuery parses the query params of a request and returns a ListOptions.
 func ParseQuery(apiOp *types.APIRequest) *ListOptions {
 	chunkSize := getLimit(apiOp)
@@ -78,11 +87,22 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions {
 	if sortKey != "" {
 		sort.field = strings.Split(sortKey, ".")
 	}
+	var err error
+	pagination := Pagination{}
+	pagination.pageSize, err = strconv.Atoi(q.Get(pageSizeParam))
+	if err != nil {
+		pagination.pageSize = 0
+	}
+	pagination.page, err = strconv.Atoi(q.Get(pageParam))
+	if err != nil {
+		pagination.page = 1
+	}
 	return &ListOptions{
-		ChunkSize: chunkSize,
-		Resume:    cont,
-		Filters:   filterOpts,
-		Sort:      sort,
+		ChunkSize:  chunkSize,
+		Resume:     cont,
+		Filters:    filterOpts,
+		Sort:       sort,
+		Pagination: pagination,
 	}
 }
 
@@ -196,3 +216,26 @@ func SortList(list []unstructured.Unstructured, s Sort) []unstructured.Unstructu
 	})
 	return list
 }
+
+// PaginateList returns a subset of the result based on the pagination criteria as well as the total number of pages the caller can expect.
+func PaginateList(list []unstructured.Unstructured, p Pagination) ([]unstructured.Unstructured, int) {
+	if p.pageSize <= 0 {
+		return list, 0
+	}
+	page := p.page - 1
+	if p.page < 1 {
+		page = 0
+	}
+	pages := len(list) / p.pageSize
+	if len(list)%p.pageSize != 0 {
+		pages++
+	}
+	offset := p.pageSize * page
+	if offset > len(list) {
+		return []unstructured.Unstructured{}, pages
+	}
+	if offset+p.pageSize > len(list) {
+		return list[offset:], pages
+	}
+	return list[offset : offset+p.pageSize], pages
+}

pkg/stores/partition/listprocessor/processor_test.go

@@ -1115,3 +1115,202 @@ func TestSortList(t *testing.T) {
 		})
 	}
 }
+
+func TestPaginateList(t *testing.T) {
+	objects := []unstructured.Unstructured{
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "fuji",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "honeycrisp",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "granny-smith",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "red-delicious",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "crispin",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "bramley",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "golden-delicious",
+				},
+			},
+		},
+		{
+			Object: map[string]interface{}{
+				"kind": "apple",
+				"metadata": map[string]interface{}{
+					"name": "macintosh",
+				},
+			},
+		},
+	}
+	tests := []struct {
+		name       string
+		objects    []unstructured.Unstructured
+		pagination Pagination
+		want       []unstructured.Unstructured
+		wantPages  int
+	}{
+		{
+			name:    "pagesize=3, page=unset",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+			},
+			want:      objects[:3],
+			wantPages: 3,
+		},
+		{
+			name:    "pagesize=3, page=1",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+				page:     1,
+			},
+			want:      objects[:3],
+			wantPages: 3,
+		},
+		{
+			name:    "pagesize=3, page=2",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+				page:     2,
+			},
+			want:      objects[3:6],
+			wantPages: 3,
+		},
+		{
+			name:    "pagesize=3, page=last",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+				page:     3,
+			},
+			want:      objects[6:],
+			wantPages: 3,
+		},
+		{
+			name:    "pagesize=3, page>last",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+				page:     37,
+			},
+			want:      []unstructured.Unstructured{},
+			wantPages: 3,
+		},
+		{
+			name:    "pagesize=3, page<0",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 3,
+				page:     -4,
+			},
+			want:      objects[:3],
+			wantPages: 3,
+		},
+		{
+			name:       "pagesize=0",
+			objects:    objects,
+			pagination: Pagination{},
+			want:       objects,
+			wantPages:  0,
+		},
+		{
+			name:    "pagesize=-1",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: -7,
+			},
+			want:      objects,
+			wantPages: 0,
+		},
+		{
+			name:    "even page size, even list size",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 2,
+				page:     2,
+			},
+			want:      objects[2:4],
+			wantPages: 4,
+		},
+		{
+			name:    "even page size, odd list size",
+			objects: objects[1:],
+			pagination: Pagination{
+				pageSize: 2,
+				page:     2,
+			},
+			want:      objects[3:5],
+			wantPages: 4,
+		},
+		{
+			name:    "odd page size, even list size",
+			objects: objects,
+			pagination: Pagination{
+				pageSize: 5,
+				page:     2,
+			},
+			want:      objects[5:],
+			wantPages: 2,
+		},
+		{
+			name:    "odd page size, odd list size",
+			objects: objects[1:],
+			pagination: Pagination{
+				pageSize: 3,
+				page:     2,
+			},
+			want:      objects[4:7],
+			wantPages: 3,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, gotPages := PaginateList(test.objects, test.pagination)
+			assert.Equal(t, test.want, got)
+			assert.Equal(t, test.wantPages, gotPages)
+		})
+	}
+}

pkg/stores/partition/store.go

@@ -129,6 +129,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
 
 	list := listprocessor.FilterList(stream, opts.Filters)
 	list = listprocessor.SortList(list, opts.Sort)
+	result.Count = len(list)
+	list, pages := listprocessor.PaginateList(list, opts.Pagination)
 
 	for _, item := range list {
 		item := item
@@ -137,6 +139,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
 
 	result.Revision = lister.Revision()
 	result.Continue = lister.Continue()
+	result.Pages = pages
+
 	return result, lister.Err()
 }
 

pkg/stores/partition/store_test.go

@@ -42,6 +42,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 1,
 					Objects: []types.APIObject{
 						newApple("fuji").toObj(),
 					},
@@ -71,18 +72,21 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count:    1,
 					Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))),
 					Objects: []types.APIObject{
 						newApple("fuji").toObj(),
 					},
 				},
 				{
+					Count:    1,
 					Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))),
 					Objects: []types.APIObject{
 						newApple("granny-smith").toObj(),
 					},
 				},
 				{
+					Count: 1,
 					Objects: []types.APIObject{
 						newApple("crispin").toObj(),
 					},
@@ -121,6 +125,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 2,
 					Objects: []types.APIObject{
 						newApple("granny-smith").toObj(),
 						newApple("crispin").toObj(),
@@ -176,6 +181,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count:    3,
 					Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)),
 					Objects: []types.APIObject{
 						newApple("fuji").toObj(),
@@ -184,6 +190,7 @@ func TestList(t *testing.T) {
 					},
 				},
 				{
+					Count:    3,
 					Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)),
 					Objects: []types.APIObject{
 						newApple("bramley").toObj(),
@@ -192,6 +199,7 @@ func TestList(t *testing.T) {
 					},
 				},
 				{
+					Count: 1,
 					Objects: []types.APIObject{
 						newApple("red-delicious").toObj(),
 					},
@@ -221,12 +229,14 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 2,
 					Objects: []types.APIObject{
 						newApple("granny-smith").toObj(),
 						newApple("bramley").toObj(),
 					},
 				},
 				{
+					Count: 1,
 					Objects: []types.APIObject{
 						newApple("bramley").toObj(),
 					},
@@ -270,6 +280,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 3,
 					Objects: []types.APIObject{
 						newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(),
 						newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(),
@@ -301,6 +312,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 4,
 					Objects: []types.APIObject{
 						newApple("bramley").toObj(),
 						newApple("crispin").toObj(),
@@ -309,6 +321,7 @@ func TestList(t *testing.T) {
 					},
 				},
 				{
+					Count: 4,
 					Objects: []types.APIObject{
 						newApple("granny-smith").toObj(),
 						newApple("fuji").toObj(),
@@ -350,6 +363,7 @@ func TestList(t *testing.T) {
 			},
 			want: []types.APIObjectList{
 				{
+					Count: 2,
 					Objects: []types.APIObject{
 						newApple("crispin").toObj(),
 						newApple("granny-smith").toObj(),