diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go new file mode 100644 index 0000000..5ecbe92 --- /dev/null +++ b/pkg/stores/partition/listprocessor/processor.go @@ -0,0 +1,150 @@ +// Package listprocessor contains methods for filtering, sorting, and paginating lists of objects. +package listprocessor + +import ( + "strconv" + "strings" + + "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/wrangler/pkg/data" + "github.com/rancher/wrangler/pkg/data/convert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + defaultLimit = 100000 + continueParam = "continue" + limitParam = "limit" + filterParam = "filter" +) + +// ListOptions represents the query parameters that may be included in a list request. +type ListOptions struct { + ChunkSize int + Resume string + Filters []Filter +} + +// Filter represents a field to filter by. +// A subfield in an object is represented in a request query using . notation, e.g. 'metadata.name'. +// The subfield is internally represented as a slice, e.g. [metadata, name]. +type Filter struct { + field []string + match string +} + +// ParseQuery parses the query params of a request and returns a ListOptions. +func ParseQuery(apiOp *types.APIRequest) *ListOptions { + chunkSize := getLimit(apiOp) + q := apiOp.Request.URL.Query() + cont := q.Get(continueParam) + filterParams := q[filterParam] + filterOpts := []Filter{} + for _, filters := range filterParams { + filter := strings.Split(filters, "=") + if len(filter) != 2 { + continue + } + filterOpts = append(filterOpts, Filter{field: strings.Split(filter[0], "."), match: filter[1]}) + } + return &ListOptions{ + ChunkSize: chunkSize, + Resume: cont, + Filters: filterOpts, + } +} + +// getLimit extracts the limit parameter from the request or sets a default of 100000. +// Since a default is always set, this implies that clients must always be +// aware that the list may be incomplete. +func getLimit(apiOp *types.APIRequest) int { + limitString := apiOp.Request.URL.Query().Get(limitParam) + limit, err := strconv.Atoi(limitString) + if err != nil { + limit = 0 + } + if limit <= 0 { + limit = defaultLimit + } + return limit +} + +// FilterList accepts a channel of unstructured objects and a slice of filters and returns the filtered list. +// Filters are ANDed together. +func FilterList(list <-chan []unstructured.Unstructured, filters []Filter) []unstructured.Unstructured { + result := []unstructured.Unstructured{} + for items := range list { + for _, item := range items { + if len(filters) == 0 { + result = append(result, item) + continue + } + if matchesAll(item.Object, filters) { + result = append(result, item) + } + } + } + return result +} + +func matchesOne(obj map[string]interface{}, filter Filter) bool { + var objValue interface{} + var ok bool + subField := []string{} + for !ok && len(filter.field) > 0 { + objValue, ok = data.GetValue(obj, filter.field...) + if !ok { + subField = append(subField, filter.field[len(filter.field)-1]) + filter.field = filter.field[:len(filter.field)-1] + } + } + if !ok { + return false + } + switch typedVal := objValue.(type) { + case string, int, bool: + if len(subField) > 0 { + return false + } + stringVal := convert.ToString(typedVal) + if strings.Contains(stringVal, filter.match) { + return true + } + case []interface{}: + filter = Filter{field: subField, match: filter.match} + if matchesAny(typedVal, filter) { + return true + } + } + return false +} + +func matchesAny(obj []interface{}, filter Filter) bool { + for _, v := range obj { + switch typedItem := v.(type) { + case string, int, bool: + stringVal := convert.ToString(typedItem) + if strings.Contains(stringVal, filter.match) { + return true + } + case map[string]interface{}: + if matchesOne(typedItem, filter) { + return true + } + case []interface{}: + if matchesAny(typedItem, filter) { + return true + } + } + } + return false +} + +func matchesAll(obj map[string]interface{}, filters []Filter) bool { + for _, f := range filters { + if !matchesOne(obj, f) { + return false + } + } + return true +} diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go new file mode 100644 index 0000000..5757b7a --- /dev/null +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -0,0 +1,798 @@ +package listprocessor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestFilterList(t *testing.T) { + tests := []struct { + name string + objects [][]unstructured.Unstructured + filters []Filter + want []unstructured.Unstructured + }{ + { + name: "single filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "multi filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + }, + { + field: []string{"metadata", "name"}, + match: "honey", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "no matches", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "purple", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "no filters", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{}, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "filter field does not match", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"spec", "volumes"}, + match: "hostPath", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "filter subfield does not match", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "productType"}, + match: "tablet", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "almost valid filter key", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "color", "shade"}, + match: "green", + }, + }, + want: []unstructured.Unstructured{}, + }, + { + name: "match string array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "pink", + "red", + "green", + "yellow", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "blue", + "red", + "black", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "yellow", + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "colors"}, + match: "yellow", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "pink", + "red", + "green", + "yellow", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "yellow", + }, + }, + }, + }, + }, + }, + { + name: "match object array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "fuji", + "color": "pink", + }, + map[string]interface{}{ + "name": "granny-smith", + "color": "green", + }, + map[string]interface{}{ + "name": "red-delicious", + "color": "red", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "blueberry", + "color": "blue", + }, + map[string]interface{}{ + "name": "raspberry", + "color": "red", + }, + map[string]interface{}{ + "name": "blackberry", + "color": "black", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "cavendish", + "color": "yellow", + }, + map[string]interface{}{ + "name": "plantain", + "color": "green", + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "varieties", "color"}, + match: "red", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "fuji", + "color": "pink", + }, + map[string]interface{}{ + "name": "granny-smith", + "color": "green", + }, + map[string]interface{}{ + "name": "red-delicious", + "color": "red", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "blueberry", + "color": "blue", + }, + map[string]interface{}{ + "name": "raspberry", + "color": "red", + }, + map[string]interface{}{ + "name": "blackberry", + "color": "black", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "pink", + "green", + "red", + "purple", + }, + []interface{}{ + "fuji", + "granny-smith", + "red-delicious", + "black-diamond", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "blue", + "red", + "black", + }, + []interface{}{ + "blueberry", + "raspberry", + "blackberry", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "yellow", + "green", + }, + []interface{}{ + "cavendish", + "plantain", + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "attributes"}, + match: "black", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "pink", + "green", + "red", + "purple", + }, + []interface{}{ + "fuji", + "granny-smith", + "red-delicious", + "black-diamond", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "blue", + "red", + "black", + }, + []interface{}{ + "blueberry", + "raspberry", + "blackberry", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested object array", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "pink": "fuji", + }, + map[string]interface{}{ + "green": "granny-smith", + }, + map[string]interface{}{ + "pink": "honeycrisp", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "blue": "blueberry", + }, + map[string]interface{}{ + "red": "raspberry", + }, + map[string]interface{}{ + "black": "blackberry", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "yellow": "cavendish", + }, + map[string]interface{}{ + "green": "plantain", + }, + }, + }, + }, + }, + }, + }, + }, + filters: []Filter{ + { + field: []string{"data", "attributes", "green"}, + match: "plantain", + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "yellow": "cavendish", + }, + map[string]interface{}{ + "green": "plantain", + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ch := make(chan []unstructured.Unstructured) + go func() { + for _, o := range test.objects { + ch <- o + } + close(ch) + }() + got := FilterList(ch, test.filters) + assert.Equal(t, test.want, got) + }) + } +} diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index ccacd08..fcdecb0 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -5,11 +5,11 @@ package partition import ( "context" "fmt" - "net/http" "reflect" "strconv" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/steve/pkg/stores/partition/listprocessor" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,8 +18,6 @@ import ( "k8s.io/apimachinery/pkg/watch" ) -const defaultLimit = 100000 - // Partitioner is an interface for interacting with partitions. type Partitioner interface { Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error) @@ -122,19 +120,18 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP Partitions: partitions, } - resume := apiOp.Request.URL.Query().Get("continue") - limit := getLimit(apiOp.Request) + opts := listprocessor.ParseQuery(apiOp) - list, err := lister.List(apiOp.Context(), limit, resume) + stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume) if err != nil { return result, err } - for items := range list { - for _, item := range items { - item := item - result.Objects = append(result.Objects, toAPI(schema, &item)) - } + list := listprocessor.FilterList(stream, opts.Filters) + + for _, item := range list { + item := item + result.Objects = append(result.Objects, toAPI(schema, &item)) } result.Revision = lister.Revision() @@ -213,21 +210,6 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types return response, nil } -// getLimit extracts the limit parameter from the request or sets a default of 100000. -// Since a default is always set, this implies that clients must always be -// aware that the list may be incomplete. -func getLimit(req *http.Request) int { - limitString := req.URL.Query().Get("limit") - limit, err := strconv.Atoi(limitString) - if err != nil { - limit = 0 - } - if limit <= 0 { - limit = defaultLimit - } - return limit -} - func toAPI(schema *types.APISchema, obj runtime.Object) types.APIObject { if obj == nil || reflect.ValueOf(obj).IsNil() { return types.APIObject{} diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index c08775f..a6577a2 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -198,6 +198,86 @@ func TestList(t *testing.T) { }, }, }, + { + name: "with filters", + apiOps: []*types.APIRequest{ + newRequest("filter=data.color=green"), + newRequest("filter=data.color=green&filter=metadata.name=bramley"), + }, + partitions: []Partition{ + mockPartition{ + name: "all", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "all": { + Items: []unstructured.Unstructured{ + newApple("fuji").Unstructured, + newApple("granny-smith").Unstructured, + newApple("bramley").Unstructured, + newApple("crispin").Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("granny-smith").toObj(), + newApple("bramley").toObj(), + }, + }, + { + Objects: []types.APIObject{ + newApple("bramley").toObj(), + }, + }, + }, + }, + { + name: "multi-partition with filters", + apiOps: []*types.APIRequest{ + newRequest("filter=data.category=baking"), + }, + partitions: []Partition{ + mockPartition{ + name: "pink", + }, + mockPartition{ + name: "green", + }, + mockPartition{ + name: "yellow", + }, + }, + objects: map[string]*unstructured.UnstructuredList{ + "pink": { + Items: []unstructured.Unstructured{ + newApple("fuji").with(map[string]string{"category": "eating"}).Unstructured, + newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).Unstructured, + }, + }, + "green": { + Items: []unstructured.Unstructured{ + newApple("granny-smith").with(map[string]string{"category": "baking"}).Unstructured, + newApple("bramley").with(map[string]string{"category": "eating"}).Unstructured, + }, + }, + "yellow": { + Items: []unstructured.Unstructured{ + newApple("crispin").with(map[string]string{"category": "baking"}).Unstructured, + }, + }, + }, + want: []types.APIObjectList{ + { + Objects: []types.APIObject{ + newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(), + newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(), + newApple("crispin").with(map[string]string{"category": "baking"}).toObj(), + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -353,3 +433,10 @@ func (a apple) toObj() types.APIObject { Object: &a.Unstructured, } } + +func (a apple) with(data map[string]string) apple { + for k, v := range data { + a.Object["data"].(map[string]interface{})[k] = v + } + return a +}