From 1d4a1309742690406369b7c8666cfaf8a593970e Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Fri, 28 Oct 2022 15:17:31 -0700 Subject: [PATCH 01/11] Add unit tests for partition listing --- pkg/stores/partition/store_test.go | 354 +++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 pkg/stores/partition/store_test.go diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go new file mode 100644 index 0000000..41fab01 --- /dev/null +++ b/pkg/stores/partition/store_test.go @@ -0,0 +1,354 @@ +package partition + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strconv" + "testing" + + "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/wrangler/pkg/schemas" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestList(t *testing.T) { + tests := []struct { + name string + apiOps []*types.APIRequest + partitions []Partition + objects map[string]types.APIObjectList + want []types.APIObjectList + }{ + { + name: "basic", + apiOps: []*types.APIRequest{ + newRequest(""), + }, + partitions: []Partition{ + mockPartition{ + name: "all", + }, + }, + objects: map[string]types.APIObjectList{ + "all": { + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + }, + }, + { + name: "limit and continue", + apiOps: []*types.APIRequest{ + newRequest("limit=1"), + newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))))), + newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))))), + }, + partitions: []Partition{ + mockPartition{ + name: "all", + }, + }, + objects: map[string]types.APIObjectList{ + "all": { + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + newApple("crispin").toObj(), + }, + }, + }, + want: []types.APIObjectList{ + { + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))), + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))), + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Objects: []types.APIObject{ + newApple("crispin").toObj(), + }, + }, + }, + }, + { + name: "multi-partition", + apiOps: []*types.APIRequest{ + newRequest(""), + }, + partitions: []Partition{ + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + }, + objects: map[string]types.APIObjectList{ + "pink": { + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + "green": { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + "yellow": { + Objects: []types.APIObject{ + newApple("crispin").toObj(), + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("crispin").toObj(), + }, + }, + }, + }, + { + name: "multi-partition with limit and continue", + apiOps: []*types.APIRequest{ + newRequest("limit=3"), + newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)))), + newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)))), + }, + partitions: []Partition{ + mockPartition{ + name: "pink", + }, + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + mockPartition{ + name: "red", + }, + }, + objects: map[string]types.APIObjectList{ + "pink": { + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("honeycrisp").toObj(), + }, + }, + "green": { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("bramley").toObj(), + }, + }, + "yellow": { + Objects: []types.APIObject{ + newApple("crispin").toObj(), + newApple("golden-delicious").toObj(), + }, + }, + "red": { + Objects: []types.APIObject{ + newApple("red-delicious").toObj(), + }, + }, + }, + want: []types.APIObjectList{ + { + Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)), + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("honeycrisp").toObj(), + newApple("granny-smith").toObj(), + }, + }, + { + Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)), + Objects: []types.APIObject{ + newApple("bramley").toObj(), + newApple("crispin").toObj(), + newApple("golden-delicious").toObj(), + }, + }, + { + Objects: []types.APIObject{ + newApple("red-delicious").toObj(), + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + schema := &types.APISchema{Schema: &schemas.Schema{ID: "apple"}} + stores := map[string]*mockStore{} + for _, p := range test.partitions { + stores[p.Name()] = &mockStore{ + contents: test.objects[p.Name()], + } + } + store := Store{ + Partitioner: mockPartitioner{ + stores: stores, + partitions: test.partitions, + }, + } + for i, req := range test.apiOps { + got, gotErr := store.List(req, schema) + assert.Nil(t, gotErr) + assert.Equal(t, test.want[i], got) + } + }) + } +} + +type mockPartitioner struct { + stores map[string]*mockStore + partitions []Partition +} + +func (m mockPartitioner) Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error) { + panic("not implemented") +} + +func (m mockPartitioner) All(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) ([]Partition, error) { + return m.partitions, nil +} + +func (m mockPartitioner) Store(apiOp *types.APIRequest, partition Partition) (types.Store, error) { + return m.stores[partition.Name()], nil +} + +type mockPartition struct { + name string +} + +func (m mockPartition) Name() string { + return m.name +} + +type mockStore struct { + contents types.APIObjectList + partition mockPartition +} + +func (m *mockStore) List(apiOp *types.APIRequest, schema *types.APISchema) (types.APIObjectList, error) { + query, _ := url.ParseQuery(apiOp.Request.URL.RawQuery) + l := query.Get("limit") + if l == "" { + return m.contents, nil + } + i := 0 + if c := query.Get("continue"); c != "" { + start, _ := base64.StdEncoding.DecodeString(c) + for j, obj := range m.contents.Objects { + if string(start) == obj.Name() { + i = j + break + } + } + } + lInt, _ := strconv.Atoi(l) + contents := m.contents + if len(contents.Objects) > i+lInt { + contents.Continue = base64.StdEncoding.EncodeToString([]byte(contents.Objects[i+lInt].Name())) + } + if i > len(contents.Objects) { + return contents, nil + } + if i+lInt > len(contents.Objects) { + contents.Objects = contents.Objects[i:] + return contents, nil + } + contents.Objects = contents.Objects[i : i+lInt] + return contents, nil +} + +func (m *mockStore) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { + panic("not implemented") +} + +func (m *mockStore) Create(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { + panic("not implemented") +} + +func (m *mockStore) Update(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject, id string) (types.APIObject, error) { + panic("not implemented") +} + +func (m *mockStore) Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { + panic("not implemented") +} + +func (m *mockStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan types.APIEvent, error) { + panic("not implemented") +} + +var colorMap = map[string]string{ + "fuji": "pink", + "honeycrisp": "pink", + "granny-smith": "green", + "bramley": "green", + "crispin": "yellow", + "golden-delicious": "yellow", + "red-delicious": "red", +} + +func newRequest(query string) *types.APIRequest { + return &types.APIRequest{ + Request: &http.Request{ + URL: &url.URL{ + Scheme: "https", + Host: "rancher", + Path: "/apples", + RawQuery: query, + }, + }, + } +} + +type apple struct { + unstructured.Unstructured +} + +func newApple(name string) apple { + return apple{unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": name, + }, + "data": map[string]interface{}{ + "color": colorMap[name], + }, + }, + }} +} + +func (a apple) toObj() types.APIObject { + return types.APIObject{ + Type: "apple", + ID: a.Object["metadata"].(map[string]interface{})["name"].(string), + Object: &a.Unstructured, + } +} From 60d234d282428698f7e07349c6a42148e855cc98 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Fri, 21 Oct 2022 16:41:08 -0700 Subject: [PATCH 02/11] Refactor proxy store Filtering and sorting needs to operate on unstructured data. It also needs to operate after the parallel partitioner, higher in the store stack. This means that the proxy Store needs to return raw, unstructured data up to the partitioner. This change moves all conversions from unstructured Kubernetes types to apiserver objects up from the proxy store into the partitioner. --- pkg/stores/partition/parallel.go | 32 ++--- pkg/stores/partition/store.go | 137 +++++++++++++++++-- pkg/stores/partition/store_test.go | 91 ++++++------- pkg/stores/proxy/proxy_store.go | 202 ++++++++++------------------- pkg/stores/proxy/rbac_store.go | 10 +- 5 files changed, 259 insertions(+), 213 deletions(-) diff --git a/pkg/stores/partition/parallel.go b/pkg/stores/partition/parallel.go index 4e9f3e5..49f380c 100644 --- a/pkg/stores/partition/parallel.go +++ b/pkg/stores/partition/parallel.go @@ -5,9 +5,9 @@ import ( "encoding/base64" "encoding/json" - "github.com/rancher/apiserver/pkg/types" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) // Partition represents a named grouping of kubernetes resources, @@ -33,7 +33,7 @@ type ParallelPartitionLister struct { } // PartitionLister lists objects for one partition. -type PartitionLister func(ctx context.Context, partition Partition, cont string, revision string, limit int) (types.APIObjectList, error) +type PartitionLister func(ctx context.Context, partition Partition, cont string, revision string, limit int) (*unstructured.UnstructuredList, error) // Err returns the latest error encountered. func (p *ParallelPartitionLister) Err() error { @@ -72,7 +72,7 @@ func indexOrZero(partitions []Partition, name string) int { // List returns a stream of objects up to the requested limit. // If the continue token is not empty, it decodes it and returns the stream // starting at the indicated marker. -func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume string) (<-chan []types.APIObject, error) { +func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume string) (<-chan []unstructured.Unstructured, error) { var state listState if resume != "" { bytes, err := base64.StdEncoding.DecodeString(resume) @@ -88,7 +88,7 @@ func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume st } } - result := make(chan []types.APIObject) + result := make(chan []unstructured.Unstructured) go p.feeder(ctx, state, limit, result) return result, nil } @@ -120,7 +120,7 @@ type listState struct { // 100000, the result is truncated and a continue token is generated that // indicates the partition and offset for the client to start on in the next // request. -func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, limit int, result chan []types.APIObject) { +func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, limit int, result chan []unstructured.Unstructured) { var ( sem = semaphore.NewWeighted(p.Concurrency) capacity = limit @@ -183,25 +183,25 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l } if state.Revision == "" { - state.Revision = list.Revision + state.Revision = list.GetResourceVersion() } if p.revision == "" { - p.revision = list.Revision + p.revision = list.GetResourceVersion() } // We have already seen the first objects in the list, truncate up to the offset. - if state.PartitionName == partition.Name() && state.Offset > 0 && state.Offset < len(list.Objects) { - list.Objects = list.Objects[state.Offset:] + if state.PartitionName == partition.Name() && state.Offset > 0 && state.Offset < len(list.Items) { + list.Items = list.Items[state.Offset:] } // Case 1: the capacity has been reached across all goroutines but the list is still only partial, // so save the state so that the next page can be requested later. - if len(list.Objects) > capacity { - result <- list.Objects[:capacity] + if len(list.Items) > capacity { + result <- list.Items[:capacity] // save state to redo this list at this offset p.state = &listState{ - Revision: list.Revision, + Revision: list.GetResourceVersion(), PartitionName: partition.Name(), Continue: cont, Offset: capacity, @@ -210,16 +210,16 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l capacity = 0 return nil } - result <- list.Objects - capacity -= len(list.Objects) + result <- list.Items + capacity -= len(list.Items) // Case 2: all objects have been returned, we are done. - if list.Continue == "" { + if list.GetContinue() == "" { return nil } // Case 3: we started at an offset and truncated the list to skip the objects up to the offset. // We're not yet up to capacity and have not retrieved every object, // so loop again and get more data. - state.Continue = list.Continue + state.Continue = list.GetContinue() state.PartitionName = partition.Name() state.Offset = 0 } diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index 4bb5340..ccacd08 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -1,12 +1,21 @@ +// Package partition implements a store with parallel partitioning of data +// so that segmented data can be concurrently collected and returned as a single data set. package partition import ( "context" + "fmt" "net/http" + "reflect" "strconv" "github.com/rancher/apiserver/pkg/types" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" ) const defaultLimit = 100000 @@ -15,7 +24,7 @@ const defaultLimit = 100000 type Partitioner interface { Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error) All(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) ([]Partition, error) - Store(apiOp *types.APIRequest, partition Partition) (types.Store, error) + Store(apiOp *types.APIRequest, partition Partition) (UnstructuredStore, error) } // Store implements types.Store for partitions. @@ -23,7 +32,17 @@ type Store struct { Partitioner Partitioner } -func (s *Store) getStore(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (types.Store, error) { +// UnstructuredStore is like types.Store but deals in k8s unstructured objects instead of apiserver types. +type UnstructuredStore interface { + ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) + List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) + Create(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject) (*unstructured.Unstructured, error) + Update(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject, id string) (*unstructured.Unstructured, error) + Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) + Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan watch.Event, error) +} + +func (s *Store) getStore(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (UnstructuredStore, error) { p, err := s.Partitioner.Lookup(apiOp, schema, verb, id) if err != nil { return nil, err @@ -39,7 +58,11 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri return types.APIObject{}, err } - return target.Delete(apiOp, schema, id) + obj, err := target.Delete(apiOp, schema, id) + if err != nil { + return types.APIObject{}, err + } + return toAPI(schema, obj), nil } // ByID looks up a single object by its ID. @@ -49,14 +72,18 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string return types.APIObject{}, err } - return target.ByID(apiOp, schema, id) + obj, err := target.ByID(apiOp, schema, id) + if err != nil { + return types.APIObject{}, err + } + return toAPI(schema, obj), nil } func (s *Store) listPartition(ctx context.Context, apiOp *types.APIRequest, schema *types.APISchema, partition Partition, - cont string, revision string, limit int) (types.APIObjectList, error) { + cont string, revision string, limit int) (*unstructured.UnstructuredList, error) { store, err := s.Partitioner.Store(apiOp, partition) if err != nil { - return types.APIObjectList{}, err + return nil, err } req := apiOp.Clone() @@ -88,7 +115,7 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP } lister := ParallelPartitionLister{ - Lister: func(ctx context.Context, partition Partition, cont string, revision string, limit int) (types.APIObjectList, error) { + Lister: func(ctx context.Context, partition Partition, cont string, revision string, limit int) (*unstructured.UnstructuredList, error) { return s.listPartition(ctx, apiOp, schema, partition, cont, revision, limit) }, Concurrency: 3, @@ -104,7 +131,10 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP } for items := range list { - result.Objects = append(result.Objects, items...) + for _, item := range items { + item := item + result.Objects = append(result.Objects, toAPI(schema, &item)) + } } result.Revision = lister.Revision() @@ -119,7 +149,11 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty return types.APIObject{}, err } - return target.Create(apiOp, schema, data) + obj, err := target.Create(apiOp, schema, data) + if err != nil { + return types.APIObject{}, err + } + return toAPI(schema, obj), nil } // Update updates a single object in the store. @@ -129,7 +163,11 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty return types.APIObject{}, err } - return target.Update(apiOp, schema, data, id) + obj, err := target.Update(apiOp, schema, data, id) + if err != nil { + return types.APIObject{}, err + } + return toAPI(schema, obj), nil } // Watch returns a channel of events for a list or resource. @@ -159,7 +197,7 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types return err } for i := range c { - response <- i + response <- toAPIEvent(apiOp, schema, i) } return nil }) @@ -189,3 +227,80 @@ func getLimit(req *http.Request) int { } return limit } + +func toAPI(schema *types.APISchema, obj runtime.Object) types.APIObject { + if obj == nil || reflect.ValueOf(obj).IsNil() { + return types.APIObject{} + } + + if unstr, ok := obj.(*unstructured.Unstructured); ok { + obj = moveToUnderscore(unstr) + } + + apiObject := types.APIObject{ + Type: schema.ID, + Object: obj, + } + + m, err := meta.Accessor(obj) + if err != nil { + return apiObject + } + + id := m.GetName() + ns := m.GetNamespace() + if ns != "" { + id = fmt.Sprintf("%s/%s", ns, id) + } + + apiObject.ID = id + return apiObject +} + +func moveToUnderscore(obj *unstructured.Unstructured) *unstructured.Unstructured { + if obj == nil { + return nil + } + + for k := range types.ReservedFields { + v, ok := obj.Object[k] + if ok { + delete(obj.Object, k) + obj.Object["_"+k] = v + } + } + + return obj +} + +func toAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Event) types.APIEvent { + name := types.ChangeAPIEvent + switch event.Type { + case watch.Deleted: + name = types.RemoveAPIEvent + case watch.Added: + name = types.CreateAPIEvent + case watch.Error: + name = "resource.error" + } + + apiEvent := types.APIEvent{ + Name: name, + } + + if event.Type == watch.Error { + status, _ := event.Object.(*metav1.Status) + apiEvent.Error = fmt.Errorf(status.Message) + return apiEvent + } + + apiEvent.Object = toAPI(schema, event.Object) + + m, err := meta.Accessor(event.Object) + if err != nil { + return apiEvent + } + + apiEvent.Revision = m.GetResourceVersion() + return apiEvent +} diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index 41fab01..c08775f 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -12,6 +12,7 @@ import ( "github.com/rancher/wrangler/pkg/schemas" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/watch" ) func TestList(t *testing.T) { @@ -19,7 +20,7 @@ func TestList(t *testing.T) { name string apiOps []*types.APIRequest partitions []Partition - objects map[string]types.APIObjectList + objects map[string]*unstructured.UnstructuredList want []types.APIObjectList }{ { @@ -32,10 +33,10 @@ func TestList(t *testing.T) { name: "all", }, }, - objects: map[string]types.APIObjectList{ + objects: map[string]*unstructured.UnstructuredList{ "all": { - Objects: []types.APIObject{ - newApple("fuji").toObj(), + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, }, }, }, @@ -59,12 +60,12 @@ func TestList(t *testing.T) { name: "all", }, }, - objects: map[string]types.APIObjectList{ + objects: map[string]*unstructured.UnstructuredList{ "all": { - Objects: []types.APIObject{ - newApple("fuji").toObj(), - newApple("granny-smith").toObj(), - newApple("crispin").toObj(), + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + newApple("crispin").Unstructured, }, }, }, @@ -101,20 +102,20 @@ func TestList(t *testing.T) { name: "yellow", }, }, - objects: map[string]types.APIObjectList{ + objects: map[string]*unstructured.UnstructuredList{ "pink": { - Objects: []types.APIObject{ - newApple("fuji").toObj(), + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, }, }, "green": { - Objects: []types.APIObject{ - newApple("granny-smith").toObj(), + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, }, }, "yellow": { - Objects: []types.APIObject{ - newApple("crispin").toObj(), + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, }, }, }, @@ -148,28 +149,28 @@ func TestList(t *testing.T) { name: "red", }, }, - objects: map[string]types.APIObjectList{ + objects: map[string]*unstructured.UnstructuredList{ "pink": { - Objects: []types.APIObject{ - newApple("fuji").toObj(), - newApple("honeycrisp").toObj(), + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("honeycrisp").Unstructured, }, }, "green": { - Objects: []types.APIObject{ - newApple("granny-smith").toObj(), - newApple("bramley").toObj(), + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, }, }, "yellow": { - Objects: []types.APIObject{ - newApple("crispin").toObj(), - newApple("golden-delicious").toObj(), + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, }, }, "red": { - Objects: []types.APIObject{ - newApple("red-delicious").toObj(), + Items: []unstructured.Unstructured{ + newApple("red-delicious").Unstructured, }, }, }, @@ -235,7 +236,7 @@ func (m mockPartitioner) All(apiOp *types.APIRequest, schema *types.APISchema, v return m.partitions, nil } -func (m mockPartitioner) Store(apiOp *types.APIRequest, partition Partition) (types.Store, error) { +func (m mockPartitioner) Store(apiOp *types.APIRequest, partition Partition) (UnstructuredStore, error) { return m.stores[partition.Name()], nil } @@ -248,11 +249,11 @@ func (m mockPartition) Name() string { } type mockStore struct { - contents types.APIObjectList + contents *unstructured.UnstructuredList partition mockPartition } -func (m *mockStore) List(apiOp *types.APIRequest, schema *types.APISchema) (types.APIObjectList, error) { +func (m *mockStore) List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) { query, _ := url.ParseQuery(apiOp.Request.URL.RawQuery) l := query.Get("limit") if l == "" { @@ -261,46 +262,46 @@ func (m *mockStore) List(apiOp *types.APIRequest, schema *types.APISchema) (type i := 0 if c := query.Get("continue"); c != "" { start, _ := base64.StdEncoding.DecodeString(c) - for j, obj := range m.contents.Objects { - if string(start) == obj.Name() { + for j, obj := range m.contents.Items { + if string(start) == obj.GetName() { i = j break } } } lInt, _ := strconv.Atoi(l) - contents := m.contents - if len(contents.Objects) > i+lInt { - contents.Continue = base64.StdEncoding.EncodeToString([]byte(contents.Objects[i+lInt].Name())) + contents := m.contents.DeepCopy() + if len(contents.Items) > i+lInt { + contents.SetContinue(base64.StdEncoding.EncodeToString([]byte(contents.Items[i+lInt].GetName()))) } - if i > len(contents.Objects) { + if i > len(contents.Items) { return contents, nil } - if i+lInt > len(contents.Objects) { - contents.Objects = contents.Objects[i:] + if i+lInt > len(contents.Items) { + contents.Items = contents.Items[i:] return contents, nil } - contents.Objects = contents.Objects[i : i+lInt] + contents.Items = contents.Items[i : i+lInt] return contents, nil } -func (m *mockStore) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { +func (m *mockStore) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) { panic("not implemented") } -func (m *mockStore) Create(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { +func (m *mockStore) Create(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject) (*unstructured.Unstructured, error) { panic("not implemented") } -func (m *mockStore) Update(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject, id string) (types.APIObject, error) { +func (m *mockStore) Update(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject, id string) (*unstructured.Unstructured, error) { panic("not implemented") } -func (m *mockStore) Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { +func (m *mockStore) Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) { panic("not implemented") } -func (m *mockStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan types.APIEvent, error) { +func (m *mockStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan watch.Event, error) { panic("not implemented") } diff --git a/pkg/stores/proxy/proxy_store.go b/pkg/stores/proxy/proxy_store.go index 8299740..c3c40c8 100644 --- a/pkg/stores/proxy/proxy_store.go +++ b/pkg/stores/proxy/proxy_store.go @@ -1,3 +1,4 @@ +// Package proxy implements the proxy store, which is responsible for interfacing directly with Kubernetes. package proxy import ( @@ -8,7 +9,6 @@ import ( "io/ioutil" "net/http" "os" - "reflect" "regexp" "strconv" @@ -65,7 +65,7 @@ type RelationshipNotifier interface { OnInboundRelationshipChange(ctx context.Context, schema *types.APISchema, namespace string) <-chan *summary.Relationship } -// Store implements types.Store directly on top of kubernetes. +// Store implements partition.UnstructuredStore directly on top of kubernetes. type Store struct { clientGetter ClientGetter notifier RelationshipNotifier @@ -89,44 +89,14 @@ func NewProxyStore(clientGetter ClientGetter, notifier RelationshipNotifier, loo } // ByID looks up a single object by its ID. -func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { - result, err := s.byID(apiOp, schema, apiOp.Namespace, id) - return toAPI(schema, result), err +func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) { + return s.byID(apiOp, schema, apiOp.Namespace, id) } func decodeParams(apiOp *types.APIRequest, target runtime.Object) error { return paramCodec.DecodeParameters(apiOp.Request.URL.Query(), metav1.SchemeGroupVersion, target) } -func toAPI(schema *types.APISchema, obj runtime.Object) types.APIObject { - if obj == nil || reflect.ValueOf(obj).IsNil() { - return types.APIObject{} - } - - if unstr, ok := obj.(*unstructured.Unstructured); ok { - obj = moveToUnderscore(unstr) - } - - apiObject := types.APIObject{ - Type: schema.ID, - Object: obj, - } - - m, err := meta.Accessor(obj) - if err != nil { - return apiObject - } - - id := m.GetName() - ns := m.GetNamespace() - if ns != "" { - id = fmt.Sprintf("%s/%s", ns, id) - } - - apiObject.ID = id - return apiObject -} - func (s *Store) byID(apiOp *types.APIRequest, schema *types.APISchema, namespace, id string) (*unstructured.Unstructured, error) { k8sClient, err := metricsStore.Wrap(s.clientGetter.TableClient(apiOp, schema, namespace)) if err != nil { @@ -158,22 +128,6 @@ func moveFromUnderscore(obj map[string]interface{}) map[string]interface{} { return obj } -func moveToUnderscore(obj *unstructured.Unstructured) *unstructured.Unstructured { - if obj == nil { - return nil - } - - for k := range types.ReservedFields { - v, ok := obj.Object[k] - if ok { - delete(obj.Object, k) - obj.Object["_"+k] = v - } - } - - return obj -} - func rowToObject(obj *unstructured.Unstructured) { if obj == nil { return @@ -230,77 +184,70 @@ func tableToObjects(obj map[string]interface{}) []unstructured.Unstructured { // to list *all* resources. // With this filter, the request can be performed successfully, and only the allowed resources will // be returned in the list. -func (s *Store) ByNames(apiOp *types.APIRequest, schema *types.APISchema, names sets.String) (types.APIObjectList, error) { +func (s *Store) ByNames(apiOp *types.APIRequest, schema *types.APISchema, names sets.String) (*unstructured.UnstructuredList, error) { if apiOp.Namespace == "*" { // This happens when you grant namespaced objects with "get" by name in a clusterrolebinding. We will treat // this as an invalid situation instead of listing all objects in the cluster and filtering by name. - return types.APIObjectList{}, nil + return nil, nil } adminClient, err := s.clientGetter.TableAdminClient(apiOp, schema, apiOp.Namespace) if err != nil { - return types.APIObjectList{}, err + return nil, err } objs, err := s.list(apiOp, schema, adminClient) if err != nil { - return types.APIObjectList{}, err + return nil, err } - var filtered []types.APIObject - for _, obj := range objs.Objects { - if names.Has(obj.Name()) { + var filtered []unstructured.Unstructured + for _, obj := range objs.Items { + if names.Has(obj.GetName()) { filtered = append(filtered, obj) } } - objs.Objects = filtered + objs.Items = filtered return objs, nil } -// List returns a list of resources. -func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.APIObjectList, error) { +// List returns an unstructured list of resources. +func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) { client, err := s.clientGetter.TableClient(apiOp, schema, apiOp.Namespace) if err != nil { - return types.APIObjectList{}, err + return nil, err } return s.list(apiOp, schema, client) } -func (s *Store) list(apiOp *types.APIRequest, schema *types.APISchema, client dynamic.ResourceInterface) (types.APIObjectList, error) { +func (s *Store) list(apiOp *types.APIRequest, schema *types.APISchema, client dynamic.ResourceInterface) (*unstructured.UnstructuredList, error) { opts := metav1.ListOptions{} if err := decodeParams(apiOp, &opts); err != nil { - return types.APIObjectList{}, nil + return nil, nil } k8sClient, _ := metricsStore.Wrap(client, nil) resultList, err := k8sClient.List(apiOp, opts) if err != nil { - return types.APIObjectList{}, err + return nil, err } tableToList(resultList) - result := types.APIObjectList{ - Revision: resultList.GetResourceVersion(), - Continue: resultList.GetContinue(), - } - - for i := range resultList.Items { - result.Objects = append(result.Objects, toAPI(schema, &resultList.Items[i])) - } - - return result, nil + return resultList, nil } -func returnErr(err error, c chan types.APIEvent) { - c <- types.APIEvent{ - Name: "resource.error", - Error: err, +func returnErr(err error, c chan watch.Event) { + c <- watch.Event{ + Type: "resource.error", + Object: &metav1.Status{ + Message: err.Error(), + }, } } -func (s *Store) listAndWatch(apiOp *types.APIRequest, client dynamic.ResourceInterface, schema *types.APISchema, w types.WatchRequest, result chan types.APIEvent) { +func (s *Store) listAndWatch(apiOp *types.APIRequest, client dynamic.ResourceInterface, schema *types.APISchema, w types.WatchRequest, result chan watch.Event) { rev := w.Revision if rev == "-1" || rev == "0" { rev = "" @@ -342,7 +289,8 @@ func (s *Store) listAndWatch(apiOp *types.APIRequest, client dynamic.ResourceInt for rel := range s.notifier.OnInboundRelationshipChange(ctx, schema, apiOp.Namespace) { obj, err := s.byID(apiOp, schema, rel.Namespace, rel.Name) if err == nil { - result <- s.toAPIEvent(apiOp, schema, watch.Modified, obj) + rowToObject(obj) + result <- watch.Event{Type: watch.Modified, Object: obj} } else { logrus.Debugf("notifier watch error: %v", err) returnErr(errors.Wrapf(err, "notifier watch error: %v", err), result) @@ -363,7 +311,10 @@ func (s *Store) listAndWatch(apiOp *types.APIRequest, client dynamic.ResourceInt } continue } - result <- s.toAPIEvent(apiOp, schema, event.Type, event.Object) + if unstr, ok := event.Object.(*unstructured.Unstructured); ok { + rowToObject(unstr) + } + result <- event } return fmt.Errorf("closed") }) @@ -378,7 +329,7 @@ func (s *Store) listAndWatch(apiOp *types.APIRequest, client dynamic.ResourceInt // to list *all* resources. // With this filter, the request can be performed successfully, and only the allowed resources will // be returned in watch. -func (s *Store) WatchNames(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest, names sets.String) (chan types.APIEvent, error) { +func (s *Store) WatchNames(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest, names sets.String) (chan watch.Event, error) { adminClient, err := s.clientGetter.TableAdminClientForWatch(apiOp, schema, apiOp.Namespace) if err != nil { return nil, err @@ -388,11 +339,16 @@ func (s *Store) WatchNames(apiOp *types.APIRequest, schema *types.APISchema, w t return nil, err } - result := make(chan types.APIEvent) + result := make(chan watch.Event) go func() { defer close(result) for item := range c { - if item.Error == nil && names.Has(item.Object.Name()) { + + m, err := meta.Accessor(item.Object) + if err != nil { + return + } + if item.Type != watch.Error && names.Has(m.GetName()) { result <- item } } @@ -402,7 +358,7 @@ func (s *Store) WatchNames(apiOp *types.APIRequest, schema *types.APISchema, w t } // Watch returns a channel of events for a list or resource. -func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan types.APIEvent, error) { +func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest) (chan watch.Event, error) { client, err := s.clientGetter.TableClientForWatch(apiOp, schema, apiOp.Namespace) if err != nil { return nil, err @@ -410,8 +366,8 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, w types. return s.watch(apiOp, schema, w, client) } -func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest, client dynamic.ResourceInterface) (chan types.APIEvent, error) { - result := make(chan types.APIEvent) +func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types.WatchRequest, client dynamic.ResourceInterface) (chan watch.Event, error) { + result := make(chan watch.Event) go func() { s.listAndWatch(apiOp, client, schema, w, result) logrus.Debugf("closing watcher for %s", schema.ID) @@ -420,35 +376,8 @@ func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types. return result, nil } -func (s *Store) toAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, et watch.EventType, obj runtime.Object) types.APIEvent { - name := types.ChangeAPIEvent - switch et { - case watch.Deleted: - name = types.RemoveAPIEvent - case watch.Added: - name = types.CreateAPIEvent - } - - if unstr, ok := obj.(*unstructured.Unstructured); ok { - rowToObject(unstr) - } - - event := types.APIEvent{ - Name: name, - Object: toAPI(schema, obj), - } - - m, err := meta.Accessor(obj) - if err != nil { - return event - } - - event.Revision = m.GetResourceVersion() - return event -} - // Create creates a single object in the store. -func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, params types.APIObject) (types.APIObject, error) { +func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, params types.APIObject) (*unstructured.Unstructured, error) { var ( resp *unstructured.Unstructured ) @@ -474,22 +403,21 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, params k8sClient, err := metricsStore.Wrap(s.clientGetter.TableClient(apiOp, schema, ns)) if err != nil { - return types.APIObject{}, err + return nil, err } opts := metav1.CreateOptions{} if err := decodeParams(apiOp, &opts); err != nil { - return types.APIObject{}, err + return nil, err } resp, err = k8sClient.Create(apiOp, &unstructured.Unstructured{Object: input}, opts) rowToObject(resp) - apiObject := toAPI(schema, resp) - return apiObject, err + return resp, err } // Update updates a single object in the store. -func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, params types.APIObject, id string) (types.APIObject, error) { +func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, params types.APIObject, id string) (*unstructured.Unstructured, error) { var ( err error input = params.Data() @@ -498,13 +426,13 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, params ns := types.Namespace(input) k8sClient, err := metricsStore.Wrap(s.clientGetter.TableClient(apiOp, schema, ns)) if err != nil { - return types.APIObject{}, err + return nil, err } if apiOp.Method == http.MethodPatch { bytes, err := ioutil.ReadAll(io.LimitReader(apiOp.Request.Body, 2<<20)) if err != nil { - return types.APIObject{}, err + return nil, err } pType := apitypes.StrategicMergePatchType @@ -514,70 +442,70 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, params opts := metav1.PatchOptions{} if err := decodeParams(apiOp, &opts); err != nil { - return types.APIObject{}, err + return nil, err } if pType == apitypes.StrategicMergePatchType { data := map[string]interface{}{} if err := json.Unmarshal(bytes, &data); err != nil { - return types.APIObject{}, err + return nil, err } data = moveFromUnderscore(data) bytes, err = json.Marshal(data) if err != nil { - return types.APIObject{}, err + return nil, err } } resp, err := k8sClient.Patch(apiOp, id, pType, bytes, opts) if err != nil { - return types.APIObject{}, err + return nil, err } - return toAPI(schema, resp), nil + return resp, nil } resourceVersion := input.String("metadata", "resourceVersion") if resourceVersion == "" { - return types.APIObject{}, fmt.Errorf("metadata.resourceVersion is required for update") + return nil, fmt.Errorf("metadata.resourceVersion is required for update") } opts := metav1.UpdateOptions{} if err := decodeParams(apiOp, &opts); err != nil { - return types.APIObject{}, err + return nil, err } resp, err := k8sClient.Update(apiOp, &unstructured.Unstructured{Object: moveFromUnderscore(input)}, metav1.UpdateOptions{}) if err != nil { - return types.APIObject{}, err + return nil, err } rowToObject(resp) - return toAPI(schema, resp), nil + return resp, nil } // Delete deletes an object from a store. -func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (types.APIObject, error) { +func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id string) (*unstructured.Unstructured, error) { opts := metav1.DeleteOptions{} if err := decodeParams(apiOp, &opts); err != nil { - return types.APIObject{}, nil + return nil, nil } k8sClient, err := metricsStore.Wrap(s.clientGetter.TableClient(apiOp, schema, apiOp.Namespace)) if err != nil { - return types.APIObject{}, err + return nil, err } if err := k8sClient.Delete(apiOp, id, opts); err != nil { - return types.APIObject{}, err + return nil, err } obj, err := s.byID(apiOp, schema, apiOp.Namespace, id) if err != nil { // ignore lookup error - return types.APIObject{}, validation.ErrorCode{ + return nil, validation.ErrorCode{ Status: http.StatusNoContent, } } - return toAPI(schema, obj), nil + return obj, nil } diff --git a/pkg/stores/proxy/rbac_store.go b/pkg/stores/proxy/rbac_store.go index 1562560..f67089d 100644 --- a/pkg/stores/proxy/rbac_store.go +++ b/pkg/stores/proxy/rbac_store.go @@ -9,7 +9,9 @@ import ( "github.com/rancher/steve/pkg/attributes" "github.com/rancher/steve/pkg/stores/partition" "github.com/rancher/wrangler/pkg/kv" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/watch" ) var ( @@ -85,8 +87,8 @@ func (p *rbacPartitioner) All(apiOp *types.APIRequest, schema *types.APISchema, } } -// Store returns a proxy Store suited to listing and watching resources by partition. -func (p *rbacPartitioner) Store(apiOp *types.APIRequest, partition partition.Partition) (types.Store, error) { +// Store returns an UnstructuredStore suited to listing and watching resources by partition. +func (p *rbacPartitioner) Store(apiOp *types.APIRequest, partition partition.Partition) (partition.UnstructuredStore, error) { return &byNameOrNamespaceStore{ Store: p.proxyStore, partition: partition.(Partition), @@ -99,7 +101,7 @@ type byNameOrNamespaceStore struct { } // List returns a list of resources by partition. -func (b *byNameOrNamespaceStore) List(apiOp *types.APIRequest, schema *types.APISchema) (types.APIObjectList, error) { +func (b *byNameOrNamespaceStore) List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) { if b.partition.Passthrough { return b.Store.List(apiOp, schema) } @@ -112,7 +114,7 @@ func (b *byNameOrNamespaceStore) List(apiOp *types.APIRequest, schema *types.API } // Watch returns a channel of resources by partition. -func (b *byNameOrNamespaceStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types.WatchRequest) (chan types.APIEvent, error) { +func (b *byNameOrNamespaceStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types.WatchRequest) (chan watch.Event, error) { if b.partition.Passthrough { return b.Store.Watch(apiOp, schema, wr) } From f8eaa11d8369256f51632d8bd22fd5a4bf484227 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Mon, 24 Oct 2022 14:17:14 -0700 Subject: [PATCH 03/11] Add filtering to partition store Extend the partition store to parse query params as list filters. Filter keys use dot notation to denote the subfield of an object to filter on. Example: GET /v1/secrets?filter=metadata.name=foo Filters are ANDed together, so an object must match all filters to be included in the list. Example: GET /v1/secrets?filter=metadata.name=foo&filter=metadata.namespace=bar Arrays are searched for matching items. If any item in the array matches, the item is included in the list. Example: GET /v1/pods?filter=spec.containers.image=alpine --- .../partition/listprocessor/processor.go | 150 ++++ .../partition/listprocessor/processor_test.go | 798 ++++++++++++++++++ pkg/stores/partition/store.go | 34 +- pkg/stores/partition/store_test.go | 87 ++ 4 files changed, 1043 insertions(+), 26 deletions(-) create mode 100644 pkg/stores/partition/listprocessor/processor.go create mode 100644 pkg/stores/partition/listprocessor/processor_test.go diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go new file mode 100644 index 0000000..5ecbe92 --- /dev/null +++ b/pkg/stores/partition/listprocessor/processor.go @@ -0,0 +1,150 @@ +// Package listprocessor contains methods for filtering, sorting, and paginating lists of objects. +package listprocessor + +import ( + "strconv" + "strings" + + "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/wrangler/pkg/data" + "github.com/rancher/wrangler/pkg/data/convert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + defaultLimit = 100000 + continueParam = "continue" + limitParam = "limit" + filterParam = "filter" +) + +// ListOptions represents the query parameters that may be included in a list request. +type ListOptions struct { + ChunkSize int + Resume string + Filters []Filter +} + +// Filter represents a field to filter by. +// A subfield in an object is represented in a request query using . notation, e.g. 'metadata.name'. +// The subfield is internally represented as a slice, e.g. [metadata, name]. +type Filter struct { + field []string + match string +} + +// ParseQuery parses the query params of a request and returns a ListOptions. +func ParseQuery(apiOp *types.APIRequest) *ListOptions { + chunkSize := getLimit(apiOp) + q := apiOp.Request.URL.Query() + cont := q.Get(continueParam) + filterParams := q[filterParam] + filterOpts := []Filter{} + for _, filters := range filterParams { + filter := strings.Split(filters, "=") + if len(filter) != 2 { + continue + } + filterOpts = append(filterOpts, Filter{field: strings.Split(filter[0], "."), match: filter[1]}) + } + return &ListOptions{ + ChunkSize: chunkSize, + Resume: cont, + Filters: filterOpts, + } +} + +// getLimit extracts the limit parameter from the request or sets a default of 100000. +// Since a default is always set, this implies that clients must always be +// aware that the list may be incomplete. +func getLimit(apiOp *types.APIRequest) int { + limitString := apiOp.Request.URL.Query().Get(limitParam) + limit, err := strconv.Atoi(limitString) + if err != nil { + limit = 0 + } + if limit <= 0 { + limit = defaultLimit + } + return limit +} + +// FilterList accepts a channel of unstructured objects and a slice of filters and returns the filtered list. +// Filters are ANDed together. +func FilterList(list <-chan []unstructured.Unstructured, filters []Filter) []unstructured.Unstructured { + result := []unstructured.Unstructured{} + for items := range list { + for _, item := range items { + if len(filters) == 0 { + result = append(result, item) + continue + } + if matchesAll(item.Object, filters) { + result = append(result, item) + } + } + } + return result +} + +func matchesOne(obj map[string]interface{}, filter Filter) bool { + var objValue interface{} + var ok bool + subField := []string{} + for !ok && len(filter.field) > 0 { + objValue, ok = data.GetValue(obj, filter.field...) + if !ok { + subField = append(subField, filter.field[len(filter.field)-1]) + filter.field = filter.field[:len(filter.field)-1] + } + } + if !ok { + return false + } + switch typedVal := objValue.(type) { + case string, int, bool: + if len(subField) > 0 { + return false + } + stringVal := convert.ToString(typedVal) + if strings.Contains(stringVal, filter.match) { + return true + } + case []interface{}: + filter = Filter{field: subField, match: filter.match} + if matchesAny(typedVal, filter) { + return true + } + } + return false +} + +func matchesAny(obj []interface{}, filter Filter) bool { + for _, v := range obj { + switch typedItem := v.(type) { + case string, int, bool: + stringVal := convert.ToString(typedItem) + if strings.Contains(stringVal, filter.match) { + return true + } + case map[string]interface{}: + if matchesOne(typedItem, filter) { + return true + } + case []interface{}: + if matchesAny(typedItem, filter) { + return true + } + } + } + return false +} + +func matchesAll(obj map[string]interface{}, filters []Filter) bool { + for _, f := range filters { + if !matchesOne(obj, f) { + return false + } + } + return true +} diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go new file mode 100644 index 0000000..5757b7a --- /dev/null +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -0,0 +1,798 @@ +package listprocessor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestFilterList(t *testing.T) { + tests := []struct { + name string + objects [][]unstructured.Unstructured + filters []Filter + want []unstructured.Unstructured + }{ + { + name: "single filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "multi filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + }, + { + field: []string{"metadata", "name"}, + match: "honey", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "no matches", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "purple", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "no filters", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{}, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "filter field does not match", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"spec", "volumes"}, + match: "hostPath", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "filter subfield does not match", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "productType"}, + match: "tablet", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "almost valid filter key", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color", "shade"}, + match: "green", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "match string array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "pink", + "red", + "green", + "yellow", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "blue", + "red", + "black", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "yellow", + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "colors"}, + match: "yellow", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "pink", + "red", + "green", + "yellow", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "yellow", + }, + }, + }, + }, + }, + }, + { + name: "match object array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "fuji", + "color": "pink", + }, + map[string]interface{}{ + "name": "granny-smith", + "color": "green", + }, + map[string]interface{}{ + "name": "red-delicious", + "color": "red", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "blueberry", + "color": "blue", + }, + map[string]interface{}{ + "name": "raspberry", + "color": "red", + }, + map[string]interface{}{ + "name": "blackberry", + "color": "black", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "cavendish", + "color": "yellow", + }, + map[string]interface{}{ + "name": "plantain", + "color": "green", + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "varieties", "color"}, + match: "red", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "fuji", + "color": "pink", + }, + map[string]interface{}{ + "name": "granny-smith", + "color": "green", + }, + map[string]interface{}{ + "name": "red-delicious", + "color": "red", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "blueberry", + "color": "blue", + }, + map[string]interface{}{ + "name": "raspberry", + "color": "red", + }, + map[string]interface{}{ + "name": "blackberry", + "color": "black", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "pink", + "green", + "red", + "purple", + }, + []interface{}{ + "fuji", + "granny-smith", + "red-delicious", + "black-diamond", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "blue", + "red", + "black", + }, + []interface{}{ + "blueberry", + "raspberry", + "blackberry", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "yellow", + "green", + }, + []interface{}{ + "cavendish", + "plantain", + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "attributes"}, + match: "black", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "pink", + "green", + "red", + "purple", + }, + []interface{}{ + "fuji", + "granny-smith", + "red-delicious", + "black-diamond", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "blue", + "red", + "black", + }, + []interface{}{ + "blueberry", + "raspberry", + "blackberry", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested object array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "pink": "fuji", + }, + map[string]interface{}{ + "green": "granny-smith", + }, + map[string]interface{}{ + "pink": "honeycrisp", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "blue": "blueberry", + }, + map[string]interface{}{ + "red": "raspberry", + }, + map[string]interface{}{ + "black": "blackberry", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "yellow": "cavendish", + }, + map[string]interface{}{ + "green": "plantain", + }, + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "attributes", "green"}, + match: "plantain", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "yellow": "cavendish", + }, + map[string]interface{}{ + "green": "plantain", + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ch := make(chan []unstructured.Unstructured) + go func() { + for _, o := range test.objects { + ch <- o + } + close(ch) + }() + got := FilterList(ch, test.filters) + assert.Equal(t, test.want, got) + }) + } +} diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index ccacd08..fcdecb0 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -5,11 +5,11 @@ package partition import ( "context" "fmt" - "net/http" "reflect" "strconv" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/steve/pkg/stores/partition/listprocessor" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,8 +18,6 @@ import ( "k8s.io/apimachinery/pkg/watch" ) -const defaultLimit = 100000 - // Partitioner is an interface for interacting with partitions. type Partitioner interface { Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error) @@ -122,19 +120,18 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP Partitions: partitions, } - resume := apiOp.Request.URL.Query().Get("continue") - limit := getLimit(apiOp.Request) + opts := listprocessor.ParseQuery(apiOp) - list, err := lister.List(apiOp.Context(), limit, resume) + stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume) if err != nil { return result, err } - for items := range list { - for _, item := range items { - item := item - result.Objects = append(result.Objects, toAPI(schema, &item)) - } + list := listprocessor.FilterList(stream, opts.Filters) + + for _, item := range list { + item := item + result.Objects = append(result.Objects, toAPI(schema, &item)) } result.Revision = lister.Revision() @@ -213,21 +210,6 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types return response, nil } -// getLimit extracts the limit parameter from the request or sets a default of 100000. -// Since a default is always set, this implies that clients must always be -// aware that the list may be incomplete. -func getLimit(req *http.Request) int { - limitString := req.URL.Query().Get("limit") - limit, err := strconv.Atoi(limitString) - if err != nil { - limit = 0 - } - if limit <= 0 { - limit = defaultLimit - } - return limit -} - func toAPI(schema *types.APISchema, obj runtime.Object) types.APIObject { if obj == nil || reflect.ValueOf(obj).IsNil() { return types.APIObject{} diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index c08775f..a6577a2 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -198,6 +198,86 @@ func TestList(t *testing.T) { }, }, }, + { + name: "with filters", + apiOps: []*types.APIRequest{ + newRequest("filter=data.color=green"), + newRequest("filter=data.color=green&filter=metadata.name=bramley"), + }, + partitions: []Partition{ + mockPartition{ + name: "all", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + newApple("crispin").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("bramley").toObj(), + }, + }, + { + Objects: []types.APIObject{ + newApple("bramley").toObj(), + }, + }, + }, + }, + { + name: "multi-partition with filters", + apiOps: []*types.APIRequest{ + newRequest("filter=data.category=baking"), + }, + partitions: []Partition{ + mockPartition{ + name: "pink", + }, + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Items: []unstructured.Unstructured{ + newApple("fuji").with(map[string]string{"category": "eating"}).Unstructured, + newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).Unstructured, + }, + }, + "green": { + Items: []unstructured.Unstructured{ + newApple("granny-smith").with(map[string]string{"category": "baking"}).Unstructured, + newApple("bramley").with(map[string]string{"category": "eating"}).Unstructured, + }, + }, + "yellow": { + Items: []unstructured.Unstructured{ + newApple("crispin").with(map[string]string{"category": "baking"}).Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(), + newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(), + newApple("crispin").with(map[string]string{"category": "baking"}).toObj(), + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -353,3 +433,10 @@ func (a apple) toObj() types.APIObject { Object: &a.Unstructured, } } + +func (a apple) with(data map[string]string) apple { + for k, v := range data { + a.Object["data"].(map[string]interface{})[k] = v + } + return a +} From adecbd91226dc37a76c39be5f79f55e711b8e3ea Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Thu, 27 Oct 2022 14:11:17 -0700 Subject: [PATCH 04/11] Add sorting to partition store Extend the partition store to parse the "sort" query parameter as a sorting condition. Dot notation is used to denote the object field. Preceding the key with "-" denotes descending (reverse) order. Example sorting by name: GET /v1/secrets?sort=metadata.name Reverse sorting by name: GET /v1/secrets?sort=-metadata.name All values are converted to strings and sorted lexicographically. --- .../partition/listprocessor/processor.go | 48 +++ .../partition/listprocessor/processor_test.go | 319 ++++++++++++++++++ pkg/stores/partition/store.go | 1 + pkg/stores/partition/store_test.go | 79 +++++ 4 files changed, 447 insertions(+) diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 5ecbe92..c311d1d 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -2,6 +2,7 @@ package listprocessor import ( + "sort" "strconv" "strings" @@ -16,6 +17,7 @@ const ( continueParam = "continue" limitParam = "limit" filterParam = "filter" + sortParam = "sort" ) // ListOptions represents the query parameters that may be included in a list request. @@ -23,6 +25,7 @@ type ListOptions struct { ChunkSize int Resume string Filters []Filter + Sort Sort } // Filter represents a field to filter by. @@ -33,6 +36,25 @@ type Filter struct { match string } +// SortOrder represents whether the list should be ascending or descending. +type SortOrder int + +const ( + // ASC stands for ascending order. + ASC SortOrder = iota + // DESC stands for descending (reverse) order. + DESC +) + +// Sort represents the criteria to sort on. +// The subfield to sort by is represented in a request query using . notation, e.g. 'metadata.name'. +// The subfield is internally represented as a slice, e.g. [metadata, name]. +// The order is represented by prefixing the sort key by '-', e.g. sort=-metadata.name. +type Sort struct { + field []string + order SortOrder +} + // ParseQuery parses the query params of a request and returns a ListOptions. func ParseQuery(apiOp *types.APIRequest) *ListOptions { chunkSize := getLimit(apiOp) @@ -47,10 +69,20 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { } filterOpts = append(filterOpts, Filter{field: strings.Split(filter[0], "."), match: filter[1]}) } + sort := Sort{} + sortKey := q.Get(sortParam) + if sortKey != "" && sortKey[0] == '-' { + sort.order = DESC + sortKey = sortKey[1:] + } + if sortKey != "" { + sort.field = strings.Split(sortKey, ".") + } return &ListOptions{ ChunkSize: chunkSize, Resume: cont, Filters: filterOpts, + Sort: sort, } } @@ -148,3 +180,19 @@ func matchesAll(obj map[string]interface{}, filters []Filter) bool { } return true } + +// SortList sorts the slice by the provided sort criteria. +func SortList(list []unstructured.Unstructured, s Sort) []unstructured.Unstructured { + if len(s.field) == 0 { + return list + } + sort.Slice(list, func(i, j int) bool { + iField := convert.ToString(data.GetValueN(list[i].Object, s.field...)) + jField := convert.ToString(data.GetValueN(list[j].Object, s.field...)) + if s.order == ASC { + return iField < jField + } + return jField < iField + }) + return list +} diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go index 5757b7a..b418ff6 100644 --- a/pkg/stores/partition/listprocessor/processor_test.go +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -796,3 +796,322 @@ func TestFilterList(t *testing.T) { }) } } + +func TestSortList(t *testing.T) { + tests := []struct { + name string + objects []unstructured.Unstructured + sort Sort + want []unstructured.Unstructured + }{ + { + name: "sort metadata.name", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + field: []string{"metadata", "name"}, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "reverse sort metadata.name", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + field: []string{"metadata", "name"}, + order: DESC, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "invalid field", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + sort: Sort{ + field: []string{"data", "productType"}, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "unsorted", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + sort: Sort{}, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := SortList(test.objects, test.sort) + assert.Equal(t, test.want, got) + }) + } +} diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index fcdecb0..0b3dcc0 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -128,6 +128,7 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP } list := listprocessor.FilterList(stream, opts.Filters) + list = listprocessor.SortList(list, opts.Sort) for _, item := range list { item := item diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index a6577a2..fd65187 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -278,6 +278,85 @@ func TestList(t *testing.T) { }, }, }, + { + name: "with sorting", + apiOps: []*types.APIRequest{ + newRequest("sort=metadata.name"), + newRequest("sort=-metadata.name"), + }, + partitions: []Partition{ + mockPartition{ + name: "all", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + newApple("crispin").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("bramley").toObj(), + newApple("crispin").toObj(), + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + }, + }, + { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("fuji").toObj(), + newApple("crispin").toObj(), + newApple("bramley").toObj(), + }, + }, + }, + }, + { + name: "multi-partition sort=metadata.name", + apiOps: []*types.APIRequest{ + newRequest("sort=metadata.name"), + }, + partitions: []Partition{ + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + }, + }, + "green": { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + }, + }, + "yellow": { + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("crispin").toObj(), + newApple("granny-smith").toObj(), + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From 9f1a27db06237e51d4c0bdf54b1b67c3b31b0bfb Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Tue, 1 Nov 2022 13:00:22 -0700 Subject: [PATCH 05/11] Add pagination to partition store Extend the partition store to parse page and pagesize parameters and return a subset of list results. The total number of pages is included in the list response. Request an initial page: GET /v1/secrets?pagesize=10 or GET /v1/secrets?pagesize=10&page=1 Request subsequent pages, or arbitrary pages: GET /v1/secrets?pagesize=10&page=37 If a page number is out of bounds, an empty list is returned. --- go.mod | 2 +- go.sum | 4 +- .../partition/listprocessor/processor.go | 59 +++++- .../partition/listprocessor/processor_test.go | 199 ++++++++++++++++++ pkg/stores/partition/store.go | 4 + pkg/stores/partition/store_test.go | 14 ++ 6 files changed, 271 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 884a57c..52b7f06 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 - github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076 + github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd github.com/rancher/dynamiclistener v0.3.5 github.com/rancher/kubernetes-provider-detector v0.1.2 github.com/rancher/norman v0.0.0-20221205184727-32ef2e185b99 diff --git a/go.sum b/go.sum index f321ea8..6c0994d 100644 --- a/go.sum +++ b/go.sum @@ -502,8 +502,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076 h1:wS95KbXFI1QOVQr3Tz+qyOJ9iia1ITCnjsapxJyI/9U= -github.com/rancher/apiserver v0.0.0-20221205175736-7c507bd5c076/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA= +github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd h1:g0hNrbONfmY4lxvrD2q9KkueYYY4wKUYscm6Ih0QfQ0= +github.com/rancher/apiserver v0.0.0-20221220225852-94cba4f28cfd/go.mod h1:xwQhXv3XFxWfA6tLa4ZeaERu8ldNbyKv2sF+mT+c5WA= github.com/rancher/client-go v1.25.4-rancher1 h1:9MlBC8QbgngUkhNzMR8rZmmCIj6WNRHFOnYiwC2Kty4= github.com/rancher/client-go v1.25.4-rancher1/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A= diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index c311d1d..172312c 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -18,14 +18,17 @@ const ( limitParam = "limit" filterParam = "filter" sortParam = "sort" + pageSizeParam = "pagesize" + pageParam = "page" ) // ListOptions represents the query parameters that may be included in a list request. type ListOptions struct { - ChunkSize int - Resume string - Filters []Filter - Sort Sort + ChunkSize int + Resume string + Filters []Filter + Sort Sort + Pagination Pagination } // Filter represents a field to filter by. @@ -55,6 +58,12 @@ type Sort struct { order SortOrder } +// Pagination represents how to return paginated results. +type Pagination struct { + pageSize int + page int +} + // ParseQuery parses the query params of a request and returns a ListOptions. func ParseQuery(apiOp *types.APIRequest) *ListOptions { chunkSize := getLimit(apiOp) @@ -78,11 +87,22 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { if sortKey != "" { sort.field = strings.Split(sortKey, ".") } + var err error + pagination := Pagination{} + pagination.pageSize, err = strconv.Atoi(q.Get(pageSizeParam)) + if err != nil { + pagination.pageSize = 0 + } + pagination.page, err = strconv.Atoi(q.Get(pageParam)) + if err != nil { + pagination.page = 1 + } return &ListOptions{ - ChunkSize: chunkSize, - Resume: cont, - Filters: filterOpts, - Sort: sort, + ChunkSize: chunkSize, + Resume: cont, + Filters: filterOpts, + Sort: sort, + Pagination: pagination, } } @@ -196,3 +216,26 @@ func SortList(list []unstructured.Unstructured, s Sort) []unstructured.Unstructu }) return list } + +// PaginateList returns a subset of the result based on the pagination criteria as well as the total number of pages the caller can expect. +func PaginateList(list []unstructured.Unstructured, p Pagination) ([]unstructured.Unstructured, int) { + if p.pageSize <= 0 { + return list, 0 + } + page := p.page - 1 + if p.page < 1 { + page = 0 + } + pages := len(list) / p.pageSize + if len(list)%p.pageSize != 0 { + pages++ + } + offset := p.pageSize * page + if offset > len(list) { + return []unstructured.Unstructured{}, pages + } + if offset+p.pageSize > len(list) { + return list[offset:], pages + } + return list[offset : offset+p.pageSize], pages +} diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go index b418ff6..4e867fc 100644 --- a/pkg/stores/partition/listprocessor/processor_test.go +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -1115,3 +1115,202 @@ func TestSortList(t *testing.T) { }) } } + +func TestPaginateList(t *testing.T) { + objects := []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "red-delicious", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "crispin", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "bramley", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "golden-delicious", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "macintosh", + }, + }, + }, + } + tests := []struct { + name string + objects []unstructured.Unstructured + pagination Pagination + want []unstructured.Unstructured + wantPages int + }{ + { + name: "pagesize=3, page=unset", + objects: objects, + pagination: Pagination{ + pageSize: 3, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=3, page=1", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 1, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=3, page=2", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 2, + }, + want: objects[3:6], + wantPages: 3, + }, + { + name: "pagesize=3, page=last", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 3, + }, + want: objects[6:], + wantPages: 3, + }, + { + name: "pagesize=3, page>last", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: 37, + }, + want: []unstructured.Unstructured{}, + wantPages: 3, + }, + { + name: "pagesize=3, page<0", + objects: objects, + pagination: Pagination{ + pageSize: 3, + page: -4, + }, + want: objects[:3], + wantPages: 3, + }, + { + name: "pagesize=0", + objects: objects, + pagination: Pagination{}, + want: objects, + wantPages: 0, + }, + { + name: "pagesize=-1", + objects: objects, + pagination: Pagination{ + pageSize: -7, + }, + want: objects, + wantPages: 0, + }, + { + name: "even page size, even list size", + objects: objects, + pagination: Pagination{ + pageSize: 2, + page: 2, + }, + want: objects[2:4], + wantPages: 4, + }, + { + name: "even page size, odd list size", + objects: objects[1:], + pagination: Pagination{ + pageSize: 2, + page: 2, + }, + want: objects[3:5], + wantPages: 4, + }, + { + name: "odd page size, even list size", + objects: objects, + pagination: Pagination{ + pageSize: 5, + page: 2, + }, + want: objects[5:], + wantPages: 2, + }, + { + name: "odd page size, odd list size", + objects: objects[1:], + pagination: Pagination{ + pageSize: 3, + page: 2, + }, + want: objects[4:7], + wantPages: 3, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, gotPages := PaginateList(test.objects, test.pagination) + assert.Equal(t, test.want, got) + assert.Equal(t, test.wantPages, gotPages) + }) + } +} diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index 0b3dcc0..6d256f8 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -129,6 +129,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP list := listprocessor.FilterList(stream, opts.Filters) list = listprocessor.SortList(list, opts.Sort) + result.Count = len(list) + list, pages := listprocessor.PaginateList(list, opts.Pagination) for _, item := range list { item := item @@ -137,6 +139,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP result.Revision = lister.Revision() result.Continue = lister.Continue() + result.Pages = pages + return result, lister.Err() } diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index fd65187..4b50c25 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -42,6 +42,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 1, Objects: []types.APIObject{ newApple("fuji").toObj(), }, @@ -71,18 +72,21 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 1, Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))), Objects: []types.APIObject{ newApple("fuji").toObj(), }, }, { + Count: 1, Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))), Objects: []types.APIObject{ newApple("granny-smith").toObj(), }, }, { + Count: 1, Objects: []types.APIObject{ newApple("crispin").toObj(), }, @@ -121,6 +125,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("crispin").toObj(), @@ -176,6 +181,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 3, Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)), Objects: []types.APIObject{ newApple("fuji").toObj(), @@ -184,6 +190,7 @@ func TestList(t *testing.T) { }, }, { + Count: 3, Continue: base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)), Objects: []types.APIObject{ newApple("bramley").toObj(), @@ -192,6 +199,7 @@ func TestList(t *testing.T) { }, }, { + Count: 1, Objects: []types.APIObject{ newApple("red-delicious").toObj(), }, @@ -221,12 +229,14 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("bramley").toObj(), }, }, { + Count: 1, Objects: []types.APIObject{ newApple("bramley").toObj(), }, @@ -270,6 +280,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 3, Objects: []types.APIObject{ newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(), newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(), @@ -301,6 +312,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 4, Objects: []types.APIObject{ newApple("bramley").toObj(), newApple("crispin").toObj(), @@ -309,6 +321,7 @@ func TestList(t *testing.T) { }, }, { + Count: 4, Objects: []types.APIObject{ newApple("granny-smith").toObj(), newApple("fuji").toObj(), @@ -350,6 +363,7 @@ func TestList(t *testing.T) { }, want: []types.APIObjectList{ { + Count: 2, Objects: []types.APIObject{ newApple("crispin").toObj(), newApple("granny-smith").toObj(), From b0eb50f38d741a86841d2115adb7e53ec5f2c954 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Thu, 3 Nov 2022 15:26:49 -0700 Subject: [PATCH 06/11] Add caching to pagination Cache filtered, sorted results for fast subsequent page retrieval. Requests for cached queries need to include the list revision number along with other queries. If no specific revision is requested, a new fetch is done in order to get the latest revision. The revision is included in the list response. Example first request: GET /v1/secrets?pagesize=10 Example subsequent page request: GET /v1/secrets?pagesize=10&page=1&revision=107740 --- .../partition/listprocessor/processor.go | 38 ++++++- pkg/stores/partition/store.go | 106 +++++++++++++++++- pkg/stores/partition/store_test.go | 64 +++++++---- pkg/stores/proxy/proxy_store.go | 7 +- 4 files changed, 179 insertions(+), 36 deletions(-) diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 172312c..614a3ee 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -20,6 +20,7 @@ const ( sortParam = "sort" pageSizeParam = "pagesize" pageParam = "page" + revisionParam = "revision" ) // ListOptions represents the query parameters that may be included in a list request. @@ -29,6 +30,7 @@ type ListOptions struct { Filters []Filter Sort Sort Pagination Pagination + Revision string } // Filter represents a field to filter by. @@ -39,6 +41,12 @@ type Filter struct { match string } +// String returns the filter as a query string. +func (f Filter) String() string { + field := strings.Join(f.field, ".") + return field + "=" + f.match +} + // SortOrder represents whether the list should be ascending or descending. type SortOrder int @@ -58,12 +66,26 @@ type Sort struct { order SortOrder } +// String returns the sort parameters as a query string. +func (s Sort) String() string { + field := strings.Join(s.field, ".") + if s.order == DESC { + field = "-" + field + } + return field +} + // Pagination represents how to return paginated results. type Pagination struct { pageSize int page int } +// PageSize returns the integer page size. +func (p Pagination) PageSize() int { + return p.pageSize +} + // ParseQuery parses the query params of a request and returns a ListOptions. func ParseQuery(apiOp *types.APIRequest) *ListOptions { chunkSize := getLimit(apiOp) @@ -78,14 +100,20 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { } filterOpts = append(filterOpts, Filter{field: strings.Split(filter[0], "."), match: filter[1]}) } - sort := Sort{} + // sort the filter fields so they can be used as a cache key in the store + sort.Slice(filterOpts, func(i, j int) bool { + fieldI := strings.Join(filterOpts[i].field, ".") + fieldJ := strings.Join(filterOpts[j].field, ".") + return fieldI < fieldJ + }) + sortOpts := Sort{} sortKey := q.Get(sortParam) if sortKey != "" && sortKey[0] == '-' { - sort.order = DESC + sortOpts.order = DESC sortKey = sortKey[1:] } if sortKey != "" { - sort.field = strings.Split(sortKey, ".") + sortOpts.field = strings.Split(sortKey, ".") } var err error pagination := Pagination{} @@ -97,12 +125,14 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { if err != nil { pagination.page = 1 } + revision := q.Get(revisionParam) return &ListOptions{ ChunkSize: chunkSize, Resume: cont, Filters: filterOpts, - Sort: sort, + Sort: sortOpts, Pagination: pagination, + Revision: revision, } } diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index 6d256f8..ca9c9c7 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -5,17 +5,32 @@ package partition import ( "context" "fmt" + "os" "reflect" "strconv" + "time" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/steve/pkg/accesscontrol" "github.com/rancher/steve/pkg/stores/partition/listprocessor" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/cache" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/endpoints/request" +) + +const ( + // Number of list request entries to save before cache replacement. + // Not related to the total size in memory of the cache, as any item could take any amount of memory. + cacheSizeEnv = "CATTLE_REQUEST_CACHE_SIZE_INT" + defaultCacheSize = 1000 + // Set to non-empty to disable list request caching entirely. + cacheDisableEnv = "CATTLE_REQUEST_CACHE_DISABLED" ) // Partitioner is an interface for interacting with partitions. @@ -28,6 +43,38 @@ type Partitioner interface { // Store implements types.Store for partitions. type Store struct { Partitioner Partitioner + listCache *cache.LRUExpireCache + asl accesscontrol.AccessSetLookup +} + +// NewStore creates a types.Store implementation with a partitioner and an LRU expiring cache for list responses. +func NewStore(partitioner Partitioner, asl accesscontrol.AccessSetLookup) *Store { + cacheSize := defaultCacheSize + if v := os.Getenv(cacheSizeEnv); v != "" { + sizeInt, err := strconv.Atoi(v) + if err == nil { + cacheSize = sizeInt + } + } + s := &Store{ + Partitioner: partitioner, + asl: asl, + } + if v := os.Getenv(cacheDisableEnv); v == "" { + s.listCache = cache.NewLRUExpireCache(cacheSize) + } + return s +} + +type cacheKey struct { + chunkSize int + resume string + filters string + sort string + pageSize int + accessID string + resourcePath string + revision string } // UnstructuredStore is like types.Store but deals in k8s unstructured objects instead of apiserver types. @@ -122,13 +169,40 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP opts := listprocessor.ParseQuery(apiOp) - stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume) + key, err := s.getCacheKey(apiOp, opts) if err != nil { return result, err } - list := listprocessor.FilterList(stream, opts.Filters) - list = listprocessor.SortList(list, opts.Sort) + var list []unstructured.Unstructured + if key.revision != "" && s.listCache != nil { + cachedList, ok := s.listCache.Get(key) + if ok { + logrus.Tracef("found cached list for query %s?%s", apiOp.Request.URL.Path, apiOp.Request.URL.RawQuery) + list = cachedList.(*unstructured.UnstructuredList).Items + result.Continue = cachedList.(*unstructured.UnstructuredList).GetContinue() + } + } + if list == nil { // did not look in cache or was not found in cache + stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume) + if err != nil { + return result, err + } + list = listprocessor.FilterList(stream, opts.Filters) + list = listprocessor.SortList(list, opts.Sort) + key.revision = lister.Revision() + listToCache := &unstructured.UnstructuredList{ + Items: list, + } + c := lister.Continue() + if c != "" { + listToCache.SetContinue(c) + } + if s.listCache != nil { + s.listCache.Add(key, listToCache, 30*time.Minute) + } + result.Continue = lister.Continue() + } result.Count = len(list) list, pages := listprocessor.PaginateList(list, opts.Pagination) @@ -137,13 +211,33 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP result.Objects = append(result.Objects, toAPI(schema, &item)) } - result.Revision = lister.Revision() - result.Continue = lister.Continue() + result.Revision = key.revision result.Pages = pages - return result, lister.Err() } +// getCacheKey returns a hashable struct identifying a unique user and request. +func (s *Store) getCacheKey(apiOp *types.APIRequest, opts *listprocessor.ListOptions) (cacheKey, error) { + user, ok := request.UserFrom(apiOp.Request.Context()) + if !ok { + return cacheKey{}, fmt.Errorf("could not find user in request") + } + filters := "" + for _, f := range opts.Filters { + filters = filters + f.String() + } + return cacheKey{ + chunkSize: opts.ChunkSize, + resume: opts.Resume, + filters: filters, + sort: opts.Sort.String(), + pageSize: opts.Pagination.PageSize(), + accessID: s.asl.AccessFor(user).ID, + resourcePath: apiOp.Request.URL.Path, + revision: opts.Revision, + }, nil +} + // Create creates a single object in the store. func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { target, err := s.getStore(apiOp, schema, "create", "") diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index 4b50c25..21325d7 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -1,6 +1,7 @@ package partition import ( + "context" "encoding/base64" "fmt" "net/http" @@ -9,10 +10,13 @@ import ( "testing" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/steve/pkg/accesscontrol" "github.com/rancher/wrangler/pkg/schemas" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" ) func TestList(t *testing.T) { @@ -26,7 +30,7 @@ func TestList(t *testing.T) { { name: "basic", apiOps: []*types.APIRequest{ - newRequest(""), + newRequest("", "user1"), }, partitions: []Partition{ mockPartition{ @@ -52,9 +56,9 @@ func TestList(t *testing.T) { { name: "limit and continue", apiOps: []*types.APIRequest{ - newRequest("limit=1"), - newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith"))))))), - newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin"))))))), + newRequest("limit=1", "user1"), + newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith")))))), "user1"), + newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin")))))), "user1"), }, partitions: []Partition{ mockPartition{ @@ -96,7 +100,7 @@ func TestList(t *testing.T) { { name: "multi-partition", apiOps: []*types.APIRequest{ - newRequest(""), + newRequest("", "user1"), }, partitions: []Partition{ mockPartition{ @@ -136,9 +140,9 @@ func TestList(t *testing.T) { { name: "multi-partition with limit and continue", apiOps: []*types.APIRequest{ - newRequest("limit=3"), - newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`)))), - newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`)))), + newRequest("limit=3", "user1"), + newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`))), "user1"), + newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`))), "user1"), }, partitions: []Partition{ mockPartition{ @@ -209,8 +213,8 @@ func TestList(t *testing.T) { { name: "with filters", apiOps: []*types.APIRequest{ - newRequest("filter=data.color=green"), - newRequest("filter=data.color=green&filter=metadata.name=bramley"), + newRequest("filter=data.color=green", "user1"), + newRequest("filter=data.color=green&filter=metadata.name=bramley", "user1"), }, partitions: []Partition{ mockPartition{ @@ -246,7 +250,7 @@ func TestList(t *testing.T) { { name: "multi-partition with filters", apiOps: []*types.APIRequest{ - newRequest("filter=data.category=baking"), + newRequest("filter=data.category=baking", "user1"), }, partitions: []Partition{ mockPartition{ @@ -292,8 +296,8 @@ func TestList(t *testing.T) { { name: "with sorting", apiOps: []*types.APIRequest{ - newRequest("sort=metadata.name"), - newRequest("sort=-metadata.name"), + newRequest("sort=metadata.name", "user1"), + newRequest("sort=-metadata.name", "user1"), }, partitions: []Partition{ mockPartition{ @@ -334,7 +338,7 @@ func TestList(t *testing.T) { { name: "multi-partition sort=metadata.name", apiOps: []*types.APIRequest{ - newRequest("sort=metadata.name"), + newRequest("sort=metadata.name", "user1"), }, partitions: []Partition{ mockPartition{ @@ -381,12 +385,11 @@ func TestList(t *testing.T) { contents: test.objects[p.Name()], } } - store := Store{ - Partitioner: mockPartitioner{ - stores: stores, - partitions: test.partitions, - }, - } + asl := &mockAccessSetLookup{} + store := NewStore(mockPartitioner{ + stores: stores, + partitions: test.partitions, + }, asl) for i, req := range test.apiOps { got, gotErr := store.List(req, schema) assert.Nil(t, gotErr) @@ -488,16 +491,19 @@ var colorMap = map[string]string{ "red-delicious": "red", } -func newRequest(query string) *types.APIRequest { +func newRequest(query, username string) *types.APIRequest { return &types.APIRequest{ - Request: &http.Request{ + Request: (&http.Request{ URL: &url.URL{ Scheme: "https", Host: "rancher", Path: "/apples", RawQuery: query, }, - }, + }).WithContext(request.WithUser(context.Background(), &user.DefaultInfo{ + Name: username, + Groups: []string{"system:authenticated"}, + })), } } @@ -533,3 +539,15 @@ func (a apple) with(data map[string]string) apple { } return a } + +type mockAccessSetLookup struct{} + +func (m *mockAccessSetLookup) AccessFor(_ user.Info) *accesscontrol.AccessSet { + return &accesscontrol.AccessSet{ + ID: "aabbccdd", + } +} + +func (m *mockAccessSetLookup) PurgeUserData(_ string) { + panic("not implemented") +} diff --git a/pkg/stores/proxy/proxy_store.go b/pkg/stores/proxy/proxy_store.go index c3c40c8..21aeba6 100644 --- a/pkg/stores/proxy/proxy_store.go +++ b/pkg/stores/proxy/proxy_store.go @@ -75,14 +75,15 @@ type Store struct { func NewProxyStore(clientGetter ClientGetter, notifier RelationshipNotifier, lookup accesscontrol.AccessSetLookup) types.Store { return &errorStore{ Store: &WatchRefresh{ - Store: &partition.Store{ - Partitioner: &rbacPartitioner{ + Store: partition.NewStore( + &rbacPartitioner{ proxyStore: &Store{ clientGetter: clientGetter, notifier: notifier, }, }, - }, + lookup, + ), asl: lookup, }, } From 52e189b1ff5d3d025cea043a9a1baa5bfd9c141f Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Fri, 4 Nov 2022 15:41:43 -0700 Subject: [PATCH 07/11] Add pagination tests for partition listing --- pkg/stores/partition/store_test.go | 1385 ++++++++++++++++++++++++++-- 1 file changed, 1332 insertions(+), 53 deletions(-) diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index 21325d7..88f9056 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -2,6 +2,7 @@ package partition import ( "context" + "crypto/sha256" "encoding/base64" "fmt" "net/http" @@ -21,20 +22,37 @@ import ( func TestList(t *testing.T) { tests := []struct { - name string - apiOps []*types.APIRequest - partitions []Partition - objects map[string]*unstructured.UnstructuredList - want []types.APIObjectList + name string + apiOps []*types.APIRequest + access []map[string]string + partitions map[string][]Partition + objects map[string]*unstructured.UnstructuredList + want []types.APIObjectList + wantCache []mockCache + disableCache bool + wantListCalls []map[string]int }{ { name: "basic", apiOps: []*types.APIRequest{ newRequest("", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "all", + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -60,9 +78,22 @@ func TestList(t *testing.T) { newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith")))))), "user1"), newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin")))))), "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "all", + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -102,12 +133,19 @@ func TestList(t *testing.T) { apiOps: []*types.APIRequest{ newRequest("", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "green", + access: []map[string]string{ + { + "user1": "roleA", }, - mockPartition{ - name: "yellow", + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -144,18 +182,31 @@ func TestList(t *testing.T) { newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"green","o":1,"l":3}`))), "user1"), newRequest(fmt.Sprintf("limit=3&continue=%s", base64.StdEncoding.EncodeToString([]byte(`{"p":"red","l":3}`))), "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "pink", + access: []map[string]string{ + { + "user1": "roleA", }, - mockPartition{ - name: "green", + { + "user1": "roleA", }, - mockPartition{ - name: "yellow", + { + "user1": "roleA", }, - mockPartition{ - name: "red", + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "pink", + }, + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + mockPartition{ + name: "red", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -216,9 +267,19 @@ func TestList(t *testing.T) { newRequest("filter=data.color=green", "user1"), newRequest("filter=data.color=green&filter=metadata.name=bramley", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "all", + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -252,15 +313,22 @@ func TestList(t *testing.T) { apiOps: []*types.APIRequest{ newRequest("filter=data.category=baking", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "pink", + access: []map[string]string{ + { + "user1": "roleA", }, - mockPartition{ - name: "green", - }, - mockPartition{ - name: "yellow", + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "pink", + }, + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -299,9 +367,19 @@ func TestList(t *testing.T) { newRequest("sort=metadata.name", "user1"), newRequest("sort=-metadata.name", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "all", + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -340,12 +418,19 @@ func TestList(t *testing.T) { apiOps: []*types.APIRequest{ newRequest("sort=metadata.name", "user1"), }, - partitions: []Partition{ - mockPartition{ - name: "green", + access: []map[string]string{ + { + "user1": "roleA", }, - mockPartition{ - name: "yellow", + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, }, }, objects: map[string]*unstructured.UnstructuredList{ @@ -375,17 +460,1178 @@ func TestList(t *testing.T) { }, }, }, + { + name: "pagination", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1&page=2&revision=42", "user1"), + newRequest("pagesize=1&page=3&revision=42", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "42", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"all": 1}, + {"all": 1}, + {"all": 1}, + }, + }, + { + name: "access-change pagination", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1&page=2&revision=42", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleB", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "42", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleB"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"all": 1}, + {"all": 2}, + }, + }, + { + name: "pagination with cache disabled", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1&page=2&revision=42", "user1"), + newRequest("pagesize=1&page=3&revision=42", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "42", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + }, + }, + wantCache: []mockCache{}, + disableCache: true, + wantListCalls: []map[string]int{ + {"all": 1}, + {"all": 2}, + {"all": 3}, + }, + }, + { + name: "multi-partition pagesize=1", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1&page=2&revision=102", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "101", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + }, + }, + "green": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "102", + }, + }, + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + }, + }, + "yellow": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "103", + }, + }, + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("crispin").toObj(), + }, + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("crispin").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("crispin").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"green": 1, "yellow": 1}, + {"green": 1, "yellow": 1}, + }, + }, + { + name: "pagesize=1 & limit=2 & continue", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1&limit=2", "user1"), + newRequest("pagesize=1&page=2&limit=2", "user1"), // does not use cache + newRequest("pagesize=1&page=2&revision=42&limit=2", "user1"), // uses cache + newRequest("pagesize=1&page=3&revision=42&limit=2", "user1"), // next page from cache + newRequest(fmt.Sprintf("pagesize=1&revision=42&limit=2&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`)))))), "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "42", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + newApple("crispin").Unstructured, + newApple("red-delicious").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "42", + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Continue: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("crispin").toObj(), + }, + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 2, + resume: "", + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "continue": base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 2, + resume: "", + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "continue": base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 2, + resume: "", + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "continue": base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 2, + resume: "", + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "continue": base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 2, + resume: "", + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "continue": base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + { + chunkSize: 2, + resume: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"r":"42","p":"all","c":"%s","l":2}`, base64.StdEncoding.EncodeToString([]byte(`crispin`))))), + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("red-delicious").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"all": 2}, + {"all": 4}, + {"all": 4}, + {"all": 4}, + {"all": 5}, + }, + }, + { + name: "multi-user pagination", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1", "user2"), + newRequest("pagesize=1&page=2&revision=42", "user1"), + newRequest("pagesize=1&page=2&revision=42", "user2"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user2": "roleB", + }, + { + "user1": "roleA", + }, + { + "user2": "roleB", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + "user2": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "42", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "42", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "42", + }: { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"all": 1}, + {"all": 2}, + {"all": 2}, + {"all": 2}, + }, + }, + { + name: "multi-partition multi-user pagination", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1", "user2"), + newRequest("pagesize=1&page=2&revision=102", "user1"), + newRequest("pagesize=1&page=2&revision=103", "user2"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user2": "roleB", + }, + { + "user1": "roleA", + }, + { + "user2": "roleB", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "green", + }, + }, + "user2": { + mockPartition{ + name: "yellow", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "101", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + }, + }, + "green": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "102", + }, + }, + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + "yellow": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "103", + }, + }, + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "103", + Objects: []types.APIObject{ + newApple("crispin").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("bramley").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "103", + Objects: []types.APIObject{ + newApple("golden-delicious").toObj(), + }, + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + cacheKey{ + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "103", + }: { + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "103", + }: { + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user2", "roleB"), + resourcePath: "/apples", + revision: "103", + }: { + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"green": 1, "yellow": 0}, + {"green": 1, "yellow": 1}, + {"green": 1, "yellow": 1}, + {"green": 1, "yellow": 1}, + }, + }, + { + name: "multi-partition access-change pagination", + apiOps: []*types.APIRequest{ + newRequest("pagesize=1", "user1"), + newRequest("pagesize=1&page=2&revision=102", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleB", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "green", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "101", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + }, + }, + "green": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "102", + }, + }, + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + "yellow": { + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "103", + }, + }, + Items: []unstructured.Unstructured{ + newApple("crispin").Unstructured, + newApple("golden-delicious").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + }, + }, + { + Count: 2, + Pages: 2, + Revision: "102", + Objects: []types.APIObject{ + newApple("bramley").toObj(), + }, + }, + }, + wantCache: []mockCache{ + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + cacheKey{ + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + }, + }, + { + contents: map[cacheKey]*unstructured.UnstructuredList{ + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleA"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + { + chunkSize: 100000, + pageSize: 1, + accessID: getAccessID("user1", "roleB"), + resourcePath: "/apples", + revision: "102", + }: { + Items: []unstructured.Unstructured{ + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + }, + }, + }, + }, + }, + wantListCalls: []map[string]int{ + {"green": 1}, + {"green": 2}, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { schema := &types.APISchema{Schema: &schemas.Schema{ID: "apple"}} stores := map[string]*mockStore{} - for _, p := range test.partitions { - stores[p.Name()] = &mockStore{ - contents: test.objects[p.Name()], + for _, partitions := range test.partitions { + for _, p := range partitions { + stores[p.Name()] = &mockStore{ + contents: test.objects[p.Name()], + } } } - asl := &mockAccessSetLookup{} + asl := &mockAccessSetLookup{userRoles: test.access} + if test.disableCache { + t.Setenv("CATTLE_REQUEST_CACHE_DISABLED", "Y") + } store := NewStore(mockPartitioner{ stores: stores, partitions: test.partitions, @@ -394,6 +1640,21 @@ func TestList(t *testing.T) { got, gotErr := store.List(req, schema) assert.Nil(t, gotErr) assert.Equal(t, test.want[i], got) + if test.disableCache { + assert.Nil(t, store.listCache) + } + if len(test.wantCache) > 0 { + assert.Equal(t, len(test.wantCache[i].contents), len(store.listCache.Keys())) + for k, v := range test.wantCache[i].contents { + cachedVal, _ := store.listCache.Get(k) + assert.Equal(t, v, cachedVal) + } + } + if len(test.wantListCalls) > 0 { + for name, _ := range store.Partitioner.(mockPartitioner).stores { + assert.Equal(t, test.wantListCalls[i][name], store.Partitioner.(mockPartitioner).stores[name].called) + } + } } }) } @@ -401,7 +1662,7 @@ func TestList(t *testing.T) { type mockPartitioner struct { stores map[string]*mockStore - partitions []Partition + partitions map[string][]Partition } func (m mockPartitioner) Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error) { @@ -409,7 +1670,8 @@ func (m mockPartitioner) Lookup(apiOp *types.APIRequest, schema *types.APISchema } func (m mockPartitioner) All(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) ([]Partition, error) { - return m.partitions, nil + user, _ := request.UserFrom(apiOp.Request.Context()) + return m.partitions[user.GetName()], nil } func (m mockPartitioner) Store(apiOp *types.APIRequest, partition Partition) (UnstructuredStore, error) { @@ -427,9 +1689,11 @@ func (m mockPartition) Name() string { type mockStore struct { contents *unstructured.UnstructuredList partition mockPartition + called int } func (m *mockStore) List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) { + m.called++ query, _ := url.ParseQuery(apiOp.Request.URL.RawQuery) l := query.Get("limit") if l == "" { @@ -481,6 +1745,10 @@ func (m *mockStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, w ty panic("not implemented") } +type mockCache struct { + contents map[cacheKey]*unstructured.UnstructuredList +} + var colorMap = map[string]string{ "fuji": "pink", "honeycrisp": "pink", @@ -540,14 +1808,25 @@ func (a apple) with(data map[string]string) apple { return a } -type mockAccessSetLookup struct{} +type mockAccessSetLookup struct { + accessID string + userRoles []map[string]string +} -func (m *mockAccessSetLookup) AccessFor(_ user.Info) *accesscontrol.AccessSet { +func (m *mockAccessSetLookup) AccessFor(user user.Info) *accesscontrol.AccessSet { + userName := user.GetName() + access := getAccessID(userName, m.userRoles[0][userName]) + m.userRoles = m.userRoles[1:] return &accesscontrol.AccessSet{ - ID: "aabbccdd", + ID: access, } } func (m *mockAccessSetLookup) PurgeUserData(_ string) { panic("not implemented") } + +func getAccessID(user, role string) string { + h := sha256.Sum256([]byte(user + role)) + return string(h[:]) +} From b151f255816c8e6c36c509e15ca149a2da0d7341 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Mon, 14 Nov 2022 14:07:23 -0800 Subject: [PATCH 08/11] Add secondary sort parameter Extend the sorting functionality in the partition store to support primary and secondary sorting criteria. Sorting parameters are specified by a single 'sort' query and comma separated. Example: Sort by name and creation time: GET /v1/secrets?sort=metadata.name,metadata.creationTimestamp Reverse sort by name, normal sort by creation time: GET /v1/secrets?sort=-metadata.name,metadata.creationTimestamp Normal sort by name, reverse sort by creation time: GET /v1/secrets?sort=metadata.name,-metadata.creationTimestamp --- .../partition/listprocessor/processor.go | 66 +++- .../partition/listprocessor/processor_test.go | 320 +++++++++++++++++- pkg/stores/partition/store_test.go | 111 ++++++ 3 files changed, 476 insertions(+), 21 deletions(-) diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 614a3ee..2cff396 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -62,16 +62,26 @@ const ( // The subfield is internally represented as a slice, e.g. [metadata, name]. // The order is represented by prefixing the sort key by '-', e.g. sort=-metadata.name. type Sort struct { - field []string - order SortOrder + primaryField []string + secondaryField []string + primaryOrder SortOrder + secondaryOrder SortOrder } // String returns the sort parameters as a query string. func (s Sort) String() string { - field := strings.Join(s.field, ".") - if s.order == DESC { + field := "" + if s.primaryOrder == DESC { field = "-" + field } + field += strings.Join(s.primaryField, ".") + if len(s.secondaryField) > 0 { + field += "," + if s.secondaryOrder == DESC { + field += "-" + } + field += strings.Join(s.secondaryField, ".") + } return field } @@ -107,13 +117,27 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { return fieldI < fieldJ }) sortOpts := Sort{} - sortKey := q.Get(sortParam) - if sortKey != "" && sortKey[0] == '-' { - sortOpts.order = DESC - sortKey = sortKey[1:] - } - if sortKey != "" { - sortOpts.field = strings.Split(sortKey, ".") + sortKeys := q.Get(sortParam) + if sortKeys != "" { + sortParts := strings.SplitN(sortKeys, ",", 2) + primaryField := sortParts[0] + if primaryField != "" && primaryField[0] == '-' { + sortOpts.primaryOrder = DESC + primaryField = primaryField[1:] + } + if primaryField != "" { + sortOpts.primaryField = strings.Split(primaryField, ".") + } + if len(sortParts) > 1 { + secondaryField := sortParts[1] + if secondaryField != "" && secondaryField[0] == '-' { + sortOpts.secondaryOrder = DESC + secondaryField = secondaryField[1:] + } + if secondaryField != "" { + sortOpts.secondaryField = strings.Split(secondaryField, ".") + } + } } var err error pagination := Pagination{} @@ -233,16 +257,24 @@ func matchesAll(obj map[string]interface{}, filters []Filter) bool { // SortList sorts the slice by the provided sort criteria. func SortList(list []unstructured.Unstructured, s Sort) []unstructured.Unstructured { - if len(s.field) == 0 { + if len(s.primaryField) == 0 { return list } sort.Slice(list, func(i, j int) bool { - iField := convert.ToString(data.GetValueN(list[i].Object, s.field...)) - jField := convert.ToString(data.GetValueN(list[j].Object, s.field...)) - if s.order == ASC { - return iField < jField + leftPrime := convert.ToString(data.GetValueN(list[i].Object, s.primaryField...)) + rightPrime := convert.ToString(data.GetValueN(list[j].Object, s.primaryField...)) + if leftPrime == rightPrime && len(s.secondaryField) > 0 { + leftSecond := convert.ToString(data.GetValueN(list[i].Object, s.secondaryField...)) + rightSecond := convert.ToString(data.GetValueN(list[j].Object, s.secondaryField...)) + if s.secondaryOrder == ASC { + return leftSecond < rightSecond + } + return rightSecond < leftSecond } - return jField < iField + if s.primaryOrder == ASC { + return leftPrime < rightPrime + } + return rightPrime < leftPrime }) return list } diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go index 4e867fc..dfc5f9d 100644 --- a/pkg/stores/partition/listprocessor/processor_test.go +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -842,7 +842,7 @@ func TestSortList(t *testing.T) { }, }, sort: Sort{ - field: []string{"metadata", "name"}, + primaryField: []string{"metadata", "name"}, }, want: []unstructured.Unstructured{ { @@ -918,8 +918,8 @@ func TestSortList(t *testing.T) { }, }, sort: Sort{ - field: []string{"metadata", "name"}, - order: DESC, + primaryField: []string{"metadata", "name"}, + primaryOrder: DESC, }, want: []unstructured.Unstructured{ { @@ -995,7 +995,7 @@ func TestSortList(t *testing.T) { }, }, sort: Sort{ - field: []string{"data", "productType"}, + primaryField: []string{"data", "productType"}, }, want: []unstructured.Unstructured{ { @@ -1107,6 +1107,318 @@ func TestSortList(t *testing.T) { }, }, }, + { + name: "primary sort ascending, secondary sort ascending", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + primaryField: []string{"data", "color"}, + secondaryField: []string{"metadata", "name"}, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "primary sort ascending, secondary sort descending", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + primaryField: []string{"data", "color"}, + secondaryField: []string{"metadata", "name"}, + secondaryOrder: DESC, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "primary sort descending, secondary sort ascending", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + primaryField: []string{"data", "color"}, + primaryOrder: DESC, + secondaryField: []string{"metadata", "name"}, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "primary sort descending, secondary sort descending", + objects: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + sort: Sort{ + primaryField: []string{"data", "color"}, + primaryOrder: DESC, + secondaryField: []string{"metadata", "name"}, + secondaryOrder: DESC, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index 88f9056..f1e295f 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -413,6 +413,117 @@ func TestList(t *testing.T) { }, }, }, + { + name: "sorting with secondary sort", + apiOps: []*types.APIRequest{ + newRequest("sort=data.color,metadata.name,", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("honeycrisp").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 3, + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("fuji").toObj(), + newApple("honeycrisp").toObj(), + }, + }, + }, + }, + { + name: "sorting with missing primary sort is unsorted", + apiOps: []*types.APIRequest{ + newRequest("sort=,metadata.name", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("honeycrisp").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 3, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("honeycrisp").toObj(), + newApple("granny-smith").toObj(), + }, + }, + }, + }, + { + name: "sorting with missing secondary sort is single-column sorted", + apiOps: []*types.APIRequest{ + newRequest("sort=metadata.name,", "user1"), + }, + access: []map[string]string{ + { + "user1": "roleA", + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("honeycrisp").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Count: 3, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + newApple("honeycrisp").toObj(), + }, + }, + }, + }, { name: "multi-partition sort=metadata.name", apiOps: []*types.APIRequest{ From 1a360d705a3573ce29b24beeaf8243f13d06bf92 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Tue, 22 Nov 2022 14:22:07 -0800 Subject: [PATCH 09/11] Add README Add an initial README to explain the basics of the API and the new parameters. --- README.md | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..d06e331 --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +steve +===== + +Steve is a lightweight API proxy for Kubernetes whose aim is to create an +interface layer suitable for dashboards to efficiently interact with +Kubernetes. + +API Usage +--------- + +### Kubernetes proxy + +Requests made to `/api`, `/api/*`, `/apis/*`, `/openapi/*` and `/version` will +be proxied directly to Kubernetes. + +### /v1 API + +Steve registers all Kubernetes resources as schemas in the /v1 API. Any +endpoint can support methods GET, POST, PATCH, PUT, or DELETE, depending on +what the underlying Kubernetes endpoint supports and the user's permissions. + +* `/v1/{type}` - all cluster-scoped resources OR all resources in all + namespaces of type `{type}` that the user has access to +* `/v1/{type}/{name}` - cluster-scoped resource of type `{type}` and unique name `{name}` +* `/v1/{type}/{namespace}` - all resources of type `{type}` under namespace `{namespace}` +* `/v1/{type}/{namespace}/{name}` - resource of type `{type}` under namespace + `{namespace}` with name `{name}` unique within the namespace + +### Query parameters + +Steve supports query parameters to perform actions or process data on top of +what Kubernetes supports. + +#### `link` + +Trigger a link handler, which is registered with the schema. Examples are +calling the shell for a cluster, or following logs during cluster or catalog +operations: + +``` +GET /v1/management.cattle.io.clusters/local?link=log +``` + +#### `action` + +Trigger an action handler, which is registered with the schema. Examples are +generating a kubeconfig for a cluster, or installing an app from a catalog: + +``` +POST /v1/catalog.cattle.io.clusterrepos/rancher-partner-charts?action=install +``` + +#### `filter` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Filter results by a designated field. Filter keys use dot notation to denote +the subfield of an object to filter on. The filter value is matched as a +substring. + +Example, filtering by object name: + +``` +/v1/{type}?filter=metadata.name=foo +``` + +Filters are ANDed together, so an object must match all filters to be +included in the list. + +``` +/v1/{type}?filter=metadata.name=foo&filter=metadata.namespace=bar +``` + +Arrays are searched for matching items. If any item in the array matches, the +item is included in the list. + +``` +/v1/{type}?filter=spec.containers.image=alpine +``` + +#### `sort` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Results can be sorted lexicographically by primary and secondary columns. + +Sorting by only a primary column, for example name: + +``` +/v1/{type}?sort=metadata.name +``` + +Reverse sorting by name: + +``` +/v1/{type}?sort=-metadata.name +``` + +The secondary sort criteria is comma separated. + +Example, sorting by name and creation time in ascending order: + +``` +/v1/{type}?sort=metadata.name,metadata.creationTimestamp +``` + +Reverse sort by name, normal sort by creation time: + +``` +/v1/{type}?sort=-metadata.name,metadata.creationTimestamp +``` + +Normal sort by name, reverse sort by creation time: + +``` +/v1/{type}?sort=metadata.name,-metadata.creationTimestamp +``` + +#### `page`, `pagesize`, and `revision` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Results can be batched by pages for easier display. + +Example initial request returning a page with 10 results: + +``` +/v1/{type}?pagesize=10 +``` + +Pages are one-indexed, so this is equivalent to + +``` +/v1/{type}?pagesize=10&page=1 +``` +To retrieve subsequent pages, the page number and the list revision number must +be included in the request. This ensures the page will be retrieved from the +cache, rather than making a new request to Kubernetes. If the revision number +is omitted, a new fetch is performed in order to get the latest revision. The +revision is included in the list response. + +``` +/v1/{type}?pagezie=10&page=2&revision=107440 +``` + +The total number of pages and individual items are included in the list +response as `pages` and `count` respectively. + +If a page number is out of bounds, an empty list is returned. + +`page` and `pagesize` can be used alongside the `limit` and `continue` +parameters supported by Kubernetes. `limit` and `continue` are typically used +for server-side chunking and do not guarantee results in any order. From 7c0228e575cded2f66297e4bbdeb3abe6f3048be Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Wed, 7 Dec 2022 14:39:05 -0800 Subject: [PATCH 10/11] Use limit=-1 to disable default list limit The default chunk size in the partition was set to 100000. It could be overridden as a larger or smaller number, but not disabled altogether. This change adds the ability for users to explicitly opt out of the limit by specifying a negative number or zero. The default behavior is the same. --- README.md | 33 +++++++++++++++++++ .../partition/listprocessor/processor.go | 7 ++-- pkg/stores/partition/parallel.go | 4 +-- pkg/stores/partition/store_test.go | 12 +++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d06e331..8343c5a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,39 @@ generating a kubeconfig for a cluster, or installing an app from a catalog: POST /v1/catalog.cattle.io.clusterrepos/rancher-partner-charts?action=install ``` +#### `limit` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Set the maximum number of results to retrieve from Kubernetes. The limit is +passed on as a parameter to the Kubernetes request. The purpose of setting this +limit is to prevent a huge response from overwhelming Steve and Rancher. For +more information about setting limits, review the Kubernetes documentation on +[retrieving results in +chunks](https://kubernetes.io/docs/reference/using-api/api-concepts/#retrieving-large-results-sets-in-chunks). + +The limit controls the size of the set coming from Kubernetes, and then +filtering, sorting, and pagination are applied on that set. Because of this, if +the result set is partial, there is no guarantee that the result returned to +the client is fully sorted across the entire list, only across the returned +chunk. + +The returned response will include a `continue` token, which indicates that the +result is partial and must be used in the subsequent request to retrieve the +next chunk. + +The default limit is 100000. To override the default, set `limit=-1`. + +#### `continue` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Continue retrieving the next chunk of a partial list. The continue token is +included in the response of a limited list and indicates that the result is +partial. This token can then be used as a query parameter to retrieve the next +chunk. All chunks have been retrieved when the continue field in the response +is empty. + #### `filter` Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 2cff396..67cace3 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -161,15 +161,12 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { } // getLimit extracts the limit parameter from the request or sets a default of 100000. -// Since a default is always set, this implies that clients must always be -// aware that the list may be incomplete. +// The default limit can be explicitly disabled by setting it to zero or negative. +// If the default is accepted, clients must be aware that the list may be incomplete, and use the "continue" token to get the next chunk of results. func getLimit(apiOp *types.APIRequest) int { limitString := apiOp.Request.URL.Query().Get(limitParam) limit, err := strconv.Atoi(limitString) if err != nil { - limit = 0 - } - if limit <= 0 { limit = defaultLimit } return limit diff --git a/pkg/stores/partition/parallel.go b/pkg/stores/partition/parallel.go index 49f380c..03c8089 100644 --- a/pkg/stores/partition/parallel.go +++ b/pkg/stores/partition/parallel.go @@ -137,7 +137,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l }() for i := indexOrZero(p.Partitions, state.PartitionName); i < len(p.Partitions); i++ { - if capacity <= 0 || isDone(ctx) { + if (limit > 0 && capacity <= 0) || isDone(ctx) { break } @@ -197,7 +197,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l // Case 1: the capacity has been reached across all goroutines but the list is still only partial, // so save the state so that the next page can be requested later. - if len(list.Items) > capacity { + if limit > 0 && len(list.Items) > capacity { result <- list.Items[:capacity] // save state to redo this list at this offset p.state = &listState{ diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index f1e295f..c0b2b46 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -77,6 +77,7 @@ func TestList(t *testing.T) { newRequest("limit=1", "user1"), newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith")))))), "user1"), newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin")))))), "user1"), + newRequest("limit=-1", "user1"), }, access: []map[string]string{ { @@ -88,6 +89,9 @@ func TestList(t *testing.T) { { "user1": "roleA", }, + { + "user1": "roleA", + }, }, partitions: map[string][]Partition{ "user1": { @@ -126,6 +130,14 @@ func TestList(t *testing.T) { newApple("crispin").toObj(), }, }, + { + Count: 3, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + newApple("crispin").toObj(), + }, + }, }, }, { From fa7fb372455f93dd2a2f5c55a9026a6adad371cf Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Wed, 7 Dec 2022 17:57:14 -0800 Subject: [PATCH 11/11] Retrieve exact revision regardless of cache state Without this change, the cache is checked if a revision is specified, but the revision is ignored if the value is not found in the cache. This is a problem if by chance sequential paginated requests land on separate pods, because a different version number may be fetched on the sequent request and cause the caches to be inconsistent with one another. This change ensures that `revision` is passed as `resourceVersion` to the Kubernetes request, since `revision` has no meaning to Kubernetes. --- pkg/stores/partition/parallel.go | 4 +- pkg/stores/partition/store.go | 13 +++- pkg/stores/partition/store_test.go | 115 ++++++++++++++++++++++++++++- 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/pkg/stores/partition/parallel.go b/pkg/stores/partition/parallel.go index 03c8089..35403aa 100644 --- a/pkg/stores/partition/parallel.go +++ b/pkg/stores/partition/parallel.go @@ -72,7 +72,7 @@ func indexOrZero(partitions []Partition, name string) int { // List returns a stream of objects up to the requested limit. // If the continue token is not empty, it decodes it and returns the stream // starting at the indicated marker. -func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume string) (<-chan []unstructured.Unstructured, error) { +func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume, revision string) (<-chan []unstructured.Unstructured, error) { var state listState if resume != "" { bytes, err := base64.StdEncoding.DecodeString(resume) @@ -86,6 +86,8 @@ func (p *ParallelPartitionLister) List(ctx context.Context, limit int, resume st if state.Limit > 0 { limit = state.Limit } + } else { + state.Revision = revision } result := make(chan []unstructured.Unstructured) diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index ca9c9c7..1efbda7 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -136,7 +136,10 @@ func (s *Store) listPartition(ctx context.Context, apiOp *types.APIRequest, sche values := req.Request.URL.Query() values.Set("continue", cont) - values.Set("revision", revision) + if revision != "" && cont == "" { + values.Set("resourceVersion", revision) + values.Set("resourceVersionMatch", "Exact") // supported since k8s 1.19 + } if limit > 0 { values.Set("limit", strconv.Itoa(limit)) } else { @@ -184,11 +187,17 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP } } if list == nil { // did not look in cache or was not found in cache - stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume) + stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume, opts.Revision) if err != nil { return result, err } list = listprocessor.FilterList(stream, opts.Filters) + // Check for any errors returned during the parallel listing requests. + // We don't want to cache the list or bother with further processing if the list is empty or corrupt. + // FilterList guarantees that the stream has been consumed and the error is populated if there is any. + if lister.Err() != nil { + return result, lister.Err() + } list = listprocessor.SortList(list, opts.Sort) key.revision = lister.Revision() listToCache := &unstructured.UnstructuredList{ diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index c0b2b46..42c8b27 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -1743,7 +1743,7 @@ func TestList(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { schema := &types.APISchema{Schema: &schemas.Schema{ID: "apple"}} - stores := map[string]*mockStore{} + stores := map[string]UnstructuredStore{} for _, partitions := range test.partitions { for _, p := range partitions { stores[p.Name()] = &mockStore{ @@ -1775,7 +1775,7 @@ func TestList(t *testing.T) { } if len(test.wantListCalls) > 0 { for name, _ := range store.Partitioner.(mockPartitioner).stores { - assert.Equal(t, test.wantListCalls[i][name], store.Partitioner.(mockPartitioner).stores[name].called) + assert.Equal(t, test.wantListCalls[i][name], store.Partitioner.(mockPartitioner).stores[name].(*mockStore).called) } } } @@ -1783,8 +1783,74 @@ func TestList(t *testing.T) { } } +func TestListByRevision(t *testing.T) { + + schema := &types.APISchema{Schema: &schemas.Schema{ID: "apple"}} + asl := &mockAccessSetLookup{userRoles: []map[string]string{ + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + }} + store := NewStore(mockPartitioner{ + stores: map[string]UnstructuredStore{ + "all": &mockVersionedStore{ + versions: []mockStore{ + { + contents: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "1", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + }, + }, + }, + { + contents: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "resourceVersion": "2", + }, + }, + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + }, + }, + }, + }, + }, + }, + partitions: map[string][]Partition{ + "user1": { + mockPartition{ + name: "all", + }, + }, + }, + }, asl) + req := newRequest("", "user1") + t.Setenv("CATTLE_REQUEST_CACHE_DISABLED", "Y") + + got, gotErr := store.List(req, schema) + assert.Nil(t, gotErr) + wantVersion := "2" + assert.Equal(t, wantVersion, got.Revision) + + req = newRequest("revision=1", "user1") + got, gotErr = store.List(req, schema) + assert.Nil(t, gotErr) + wantVersion = "1" + assert.Equal(t, wantVersion, got.Revision) +} + type mockPartitioner struct { - stores map[string]*mockStore + stores map[string]UnstructuredStore partitions map[string][]Partition } @@ -1868,6 +1934,49 @@ func (m *mockStore) Watch(apiOp *types.APIRequest, schema *types.APISchema, w ty panic("not implemented") } +type mockVersionedStore struct { + mockStore + versions []mockStore +} + +func (m *mockVersionedStore) List(apiOp *types.APIRequest, schema *types.APISchema) (*unstructured.UnstructuredList, error) { + m.called++ + query, _ := url.ParseQuery(apiOp.Request.URL.RawQuery) + rv := len(m.versions) - 1 + if query.Get("resourceVersion") != "" { + rv, _ = strconv.Atoi(query.Get("resourceVersion")) + rv-- + } + l := query.Get("limit") + if l == "" { + return m.versions[rv].contents, nil + } + i := 0 + if c := query.Get("continue"); c != "" { + start, _ := base64.StdEncoding.DecodeString(c) + for j, obj := range m.versions[rv].contents.Items { + if string(start) == obj.GetName() { + i = j + break + } + } + } + lInt, _ := strconv.Atoi(l) + contents := m.versions[rv].contents.DeepCopy() + if len(contents.Items) > i+lInt { + contents.SetContinue(base64.StdEncoding.EncodeToString([]byte(contents.Items[i+lInt].GetName()))) + } + if i > len(contents.Items) { + return contents, nil + } + if i+lInt > len(contents.Items) { + contents.Items = contents.Items[i:] + return contents, nil + } + contents.Items = contents.Items[i : i+lInt] + return contents, nil +} + type mockCache struct { contents map[cacheKey]*unstructured.UnstructuredList }