From 61a39906f934ddff552f811b1eec5efd5f20f661 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Mon, 6 Mar 2023 16:10:50 -0800 Subject: [PATCH] Add support for NOT filters This change adds support for excluding results using the != operator. Example to exclude all results with name "example": ?filter=metadata.name!=example Include all results from namespace "example" but exclude those with name "example": ?filter=metadata.namespace=example&metadata.name!=example Exclude results with name "foo" OR exclude results with name "bar" (effectively includes results of both types): ?filter=metadata.name!=foo,metadata.name!=bar --- .../partition/listprocessor/processor.go | 24 +- .../partition/listprocessor/processor_test.go | 652 ++++++++++++++++++ pkg/stores/partition/store_test.go | 33 + 3 files changed, 705 insertions(+), 4 deletions(-) diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 75bdee4..a316a53 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -2,6 +2,7 @@ package listprocessor import ( + "regexp" "sort" "strconv" "strings" @@ -24,6 +25,15 @@ const ( orOp = "," ) +var opReg = regexp.MustCompile(`[!]?=`) + +type op string + +const ( + eq op = "" + notEq op = "!=" +) + // ListOptions represents the query parameters that may be included in a list request. type ListOptions struct { ChunkSize int @@ -40,6 +50,7 @@ type ListOptions struct { type Filter struct { field []string match string + op op } // String returns the filter as a query string. @@ -127,11 +138,15 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { orFilters := strings.Split(filters, orOp) orFilter := OrFilter{} for _, filter := range orFilters { - filter := strings.Split(filter, "=") + var op op + if strings.Contains(filter, "!=") { + op = "!=" + } + filter := opReg.Split(filter, -1) if len(filter) != 2 { continue } - orFilter.filters = append(orFilter.filters, Filter{field: strings.Split(filter[0], "."), match: filter[1]}) + orFilter.filters = append(orFilter.filters, Filter{field: strings.Split(filter[0], "."), match: filter[1], op: op}) } filterOpts = append(filterOpts, orFilter) } @@ -251,7 +266,7 @@ func matchesOne(obj map[string]interface{}, filter Filter) bool { return true } case []interface{}: - filter = Filter{field: subField, match: filter.match} + filter = Filter{field: subField, match: filter.match, op: filter.op} if matchesOneInList(typedVal, filter) { return true } @@ -282,7 +297,8 @@ func matchesOneInList(obj []interface{}, filter Filter) bool { func matchesAny(obj map[string]interface{}, filter OrFilter) bool { for _, f := range filter.filters { - if matches := matchesOne(obj, f); matches { + matches := matchesOne(obj, f) + if (matches && f.op == eq) || (!matches && f.op == notEq) { return true } } diff --git a/pkg/stores/partition/listprocessor/processor_test.go b/pkg/stores/partition/listprocessor/processor_test.go index 2f0ba64..a5e7c96 100644 --- a/pkg/stores/partition/listprocessor/processor_test.go +++ b/pkg/stores/partition/listprocessor/processor_test.go @@ -1070,6 +1070,658 @@ func TestFilterList(t *testing.T) { }, }, }, + { + name: "not filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + op: "!=", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "or'ed not filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + op: "!=", + }, + { + field: []string{"data", "color"}, + match: "green", + op: "!=", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "mixed or'ed filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + op: "!=", + }, + { + field: []string{"metadata", "name"}, + match: "fuji", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + { + name: "anded and or'ed mixed equality filter", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "fuji", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "granny-smith", + }, + "data": map[string]interface{}{ + "color": "green", + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"metadata", "name"}, + match: "fuji", + op: "!=", + }, + }, + }, + { + filters: []Filter{ + { + field: []string{"data", "color"}, + match: "pink", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "apple", + "metadata": map[string]interface{}{ + "name": "honeycrisp", + }, + "data": map[string]interface{}{ + "color": "pink", + }, + }, + }, + }, + }, + { + name: "match string array with not", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "pink", + "red", + "green", + "yellow", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "blue", + "red", + "black", + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "yellow", + }, + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "colors"}, + match: "yellow", + op: "!=", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "colors": []interface{}{ + "blue", + "red", + "black", + }, + }, + }, + }, + }, + }, + { + name: "match object array with not", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "fuji", + "color": "pink", + }, + map[string]interface{}{ + "name": "granny-smith", + "color": "green", + }, + map[string]interface{}{ + "name": "red-delicious", + "color": "red", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "blueberry", + "color": "blue", + }, + map[string]interface{}{ + "name": "raspberry", + "color": "red", + }, + map[string]interface{}{ + "name": "blackberry", + "color": "black", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "cavendish", + "color": "yellow", + }, + map[string]interface{}{ + "name": "plantain", + "color": "green", + }, + }, + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "varieties", "color"}, + match: "red", + op: "!=", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "varieties": []interface{}{ + map[string]interface{}{ + "name": "cavendish", + "color": "yellow", + }, + map[string]interface{}{ + "name": "plantain", + "color": "green", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested array with not", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "pink", + "green", + "red", + "purple", + }, + []interface{}{ + "fuji", + "granny-smith", + "red-delicious", + "black-diamond", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "blue", + "red", + "black", + }, + []interface{}{ + "blueberry", + "raspberry", + "blackberry", + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "yellow", + "green", + }, + []interface{}{ + "cavendish", + "plantain", + }, + }, + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "attributes"}, + match: "black", + op: "!=", + }, + }, + }, + }, + want: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + "yellow", + "green", + }, + []interface{}{ + "cavendish", + "plantain", + }, + }, + }, + }, + }, + }, + }, + { + name: "match nested object array with mixed equality", + objects: [][]unstructured.Unstructured{ + { + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "apple", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "pink": "fuji", + }, + map[string]interface{}{ + "green": "granny-smith", + }, + map[string]interface{}{ + "pink": "honeycrisp", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "berry", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "blue": "blueberry", + }, + map[string]interface{}{ + "red": "raspberry", + }, + map[string]interface{}{ + "black": "blackberry", + }, + }, + }, + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "fruit", + "metadata": map[string]interface{}{ + "name": "banana", + }, + "data": map[string]interface{}{ + "attributes": []interface{}{ + []interface{}{ + map[string]interface{}{ + "yellow": "cavendish", + }, + map[string]interface{}{ + "green": "plantain", + }, + }, + }, + }, + }, + }, + }, + }, + filters: []OrFilter{ + { + filters: []Filter{ + { + field: []string{"data", "attributes", "green"}, + match: "plantain", + op: "!=", + }, + { + field: []string{"data", "attributes", "green"}, + match: "granny-smith", + }, + }, + }, + { + filters: []Filter{ + { + field: []string{"metadata", "name"}, + match: "banana", + }, + }, + }, + }, + want: []unstructured.Unstructured{}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index 8d466a4..fdd666c 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -281,6 +281,9 @@ func TestList(t *testing.T) { newRequest("filter=data.color=green,data.color=pink", "user1"), newRequest("filter=data.color=green,data.color=pink&filter=metadata.name=fuji", "user1"), newRequest("filter=data.color=green,data.color=pink&filter=metadata.name=crispin", "user1"), + newRequest("filter=data.color!=green", "user1"), + newRequest("filter=data.color!=green,metadata.name=granny-smith", "user1"), + newRequest("filter=data.color!=green&filter=metadata.name!=crispin", "user1"), }, access: []map[string]string{ { @@ -298,6 +301,15 @@ func TestList(t *testing.T) { { "user1": "roleA", }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, + { + "user1": "roleA", + }, }, partitions: map[string][]Partition{ "user1": { @@ -347,6 +359,27 @@ func TestList(t *testing.T) { { Count: 0, }, + { + Count: 2, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("crispin").toObj(), + }, + }, + { + Count: 3, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + newApple("crispin").toObj(), + }, + }, + { + Count: 1, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + }, + }, }, }, {