mirror of
https://github.com/niusmallnan/steve.git
synced 2025-08-01 22:08:06 +00:00
Add filtering to partition store
Extend the partition store to parse query params as list filters. Filter keys use dot notation to denote the subfield of an object to filter on. Example: GET /v1/secrets?filter=metadata.name=foo Filters are ANDed together, so an object must match all filters to be included in the list. Example: GET /v1/secrets?filter=metadata.name=foo&filter=metadata.namespace=bar Arrays are searched for matching items. If any item in the array matches, the item is included in the list. Example: GET /v1/pods?filter=spec.containers.image=alpine
This commit is contained in:
Colleen Murphy
parent
60d234d282
commit
f8eaa11d83
150
pkg/stores/partition/listprocessor/processor.go
Normal file
150
pkg/stores/partition/listprocessor/processor.go
Normal file
@ -0,0 +1,150 @@
|
||||
// Package listprocessor contains methods for filtering, sorting, and paginating lists of objects.
|
||||
package listprocessor
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rancher/apiserver/pkg/types"
|
||||
"github.com/rancher/wrangler/pkg/data"
|
||||
"github.com/rancher/wrangler/pkg/data/convert"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLimit = 100000
|
||||
continueParam = "continue"
|
||||
limitParam = "limit"
|
||||
filterParam = "filter"
|
||||
)
|
||||
|
||||
// ListOptions represents the query parameters that may be included in a list request.
|
||||
type ListOptions struct {
|
||||
ChunkSize int
|
||||
Resume string
|
||||
Filters []Filter
|
||||
}
|
||||
|
||||
// Filter represents a field to filter by.
|
||||
// A subfield in an object is represented in a request query using . notation, e.g. 'metadata.name'.
|
||||
// The subfield is internally represented as a slice, e.g. [metadata, name].
|
||||
type Filter struct {
|
||||
field []string
|
||||
match string
|
||||
}
|
||||
|
||||
// ParseQuery parses the query params of a request and returns a ListOptions.
|
||||
func ParseQuery(apiOp *types.APIRequest) *ListOptions {
|
||||
chunkSize := getLimit(apiOp)
|
||||
q := apiOp.Request.URL.Query()
|
||||
cont := q.Get(continueParam)
|
||||
filterParams := q[filterParam]
|
||||
filterOpts := []Filter{}
|
||||
for _, filters := range filterParams {
|
||||
filter := strings.Split(filters, "=")
|
||||
if len(filter) != 2 {
|
||||
continue
|
||||
}
|
||||
filterOpts = append(filterOpts, Filter{field: strings.Split(filter[0], "."), match: filter[1]})
|
||||
}
|
||||
return &ListOptions{
|
||||
ChunkSize: chunkSize,
|
||||
Resume: cont,
|
||||
Filters: filterOpts,
|
||||
}
|
||||
}
|
||||
|
||||
// getLimit extracts the limit parameter from the request or sets a default of 100000.
|
||||
// Since a default is always set, this implies that clients must always be
|
||||
// aware that the list may be incomplete.
|
||||
func getLimit(apiOp *types.APIRequest) int {
|
||||
limitString := apiOp.Request.URL.Query().Get(limitParam)
|
||||
limit, err := strconv.Atoi(limitString)
|
||||
if err != nil {
|
||||
limit = 0
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = defaultLimit
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
// FilterList accepts a channel of unstructured objects and a slice of filters and returns the filtered list.
|
||||
// Filters are ANDed together.
|
||||
func FilterList(list <-chan []unstructured.Unstructured, filters []Filter) []unstructured.Unstructured {
|
||||
result := []unstructured.Unstructured{}
|
||||
for items := range list {
|
||||
for _, item := range items {
|
||||
if len(filters) == 0 {
|
||||
result = append(result, item)
|
||||
continue
|
||||
}
|
||||
if matchesAll(item.Object, filters) {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func matchesOne(obj map[string]interface{}, filter Filter) bool {
|
||||
var objValue interface{}
|
||||
var ok bool
|
||||
subField := []string{}
|
||||
for !ok && len(filter.field) > 0 {
|
||||
objValue, ok = data.GetValue(obj, filter.field...)
|
||||
if !ok {
|
||||
subField = append(subField, filter.field[len(filter.field)-1])
|
||||
filter.field = filter.field[:len(filter.field)-1]
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch typedVal := objValue.(type) {
|
||||
case string, int, bool:
|
||||
if len(subField) > 0 {
|
||||
return false
|
||||
}
|
||||
stringVal := convert.ToString(typedVal)
|
||||
if strings.Contains(stringVal, filter.match) {
|
||||
return true
|
||||
}
|
||||
case []interface{}:
|
||||
filter = Filter{field: subField, match: filter.match}
|
||||
if matchesAny(typedVal, filter) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func matchesAny(obj []interface{}, filter Filter) bool {
|
||||
for _, v := range obj {
|
||||
switch typedItem := v.(type) {
|
||||
case string, int, bool:
|
||||
stringVal := convert.ToString(typedItem)
|
||||
if strings.Contains(stringVal, filter.match) {
|
||||
return true
|
||||
}
|
||||
case map[string]interface{}:
|
||||
if matchesOne(typedItem, filter) {
|
||||
return true
|
||||
}
|
||||
case []interface{}:
|
||||
if matchesAny(typedItem, filter) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func matchesAll(obj map[string]interface{}, filters []Filter) bool {
|
||||
for _, f := range filters {
|
||||
if !matchesOne(obj, f) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
798
pkg/stores/partition/listprocessor/processor_test.go
Normal file
798
pkg/stores/partition/listprocessor/processor_test.go
Normal file
@ -0,0 +1,798 @@
|
||||
package listprocessor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
)
|
||||
|
||||
func TestFilterList(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
objects [][]unstructured.Unstructured
|
||||
filters []Filter
|
||||
want []unstructured.Unstructured
|
||||
}{
|
||||
{
|
||||
name: "single filter",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "color"},
|
||||
match: "pink",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multi filter",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "honeycrisp",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "color"},
|
||||
match: "pink",
|
||||
},
|
||||
{
|
||||
field: []string{"metadata", "name"},
|
||||
match: "honey",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "honeycrisp",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no matches",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "color"},
|
||||
match: "purple",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{},
|
||||
},
|
||||
{
|
||||
name: "no filters",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "filter field does not match",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "honeycrisp",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"spec", "volumes"},
|
||||
match: "hostPath",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{},
|
||||
},
|
||||
{
|
||||
name: "filter subfield does not match",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "fuji",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "honeycrisp",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "pink",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "productType"},
|
||||
match: "tablet",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{},
|
||||
},
|
||||
{
|
||||
name: "almost valid filter key",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "apple",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "color", "shade"},
|
||||
match: "green",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{},
|
||||
},
|
||||
{
|
||||
name: "match string array",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"colors": []interface{}{
|
||||
"pink",
|
||||
"red",
|
||||
"green",
|
||||
"yellow",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"colors": []interface{}{
|
||||
"blue",
|
||||
"red",
|
||||
"black",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"colors": []interface{}{
|
||||
"yellow",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "colors"},
|
||||
match: "yellow",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"colors": []interface{}{
|
||||
"pink",
|
||||
"red",
|
||||
"green",
|
||||
"yellow",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"colors": []interface{}{
|
||||
"yellow",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "match object array",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"varieties": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "fuji",
|
||||
"color": "pink",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
"color": "green",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "red-delicious",
|
||||
"color": "red",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"varieties": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "blueberry",
|
||||
"color": "blue",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "raspberry",
|
||||
"color": "red",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "blackberry",
|
||||
"color": "black",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"varieties": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "cavendish",
|
||||
"color": "yellow",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "plantain",
|
||||
"color": "green",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "varieties", "color"},
|
||||
match: "red",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"varieties": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "fuji",
|
||||
"color": "pink",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "granny-smith",
|
||||
"color": "green",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "red-delicious",
|
||||
"color": "red",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"varieties": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "blueberry",
|
||||
"color": "blue",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "raspberry",
|
||||
"color": "red",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "blackberry",
|
||||
"color": "black",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "match nested array",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
"pink",
|
||||
"green",
|
||||
"red",
|
||||
"purple",
|
||||
},
|
||||
[]interface{}{
|
||||
"fuji",
|
||||
"granny-smith",
|
||||
"red-delicious",
|
||||
"black-diamond",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
"blue",
|
||||
"red",
|
||||
"black",
|
||||
},
|
||||
[]interface{}{
|
||||
"blueberry",
|
||||
"raspberry",
|
||||
"blackberry",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
"yellow",
|
||||
"green",
|
||||
},
|
||||
[]interface{}{
|
||||
"cavendish",
|
||||
"plantain",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "attributes"},
|
||||
match: "black",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
"pink",
|
||||
"green",
|
||||
"red",
|
||||
"purple",
|
||||
},
|
||||
[]interface{}{
|
||||
"fuji",
|
||||
"granny-smith",
|
||||
"red-delicious",
|
||||
"black-diamond",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
"blue",
|
||||
"red",
|
||||
"black",
|
||||
},
|
||||
[]interface{}{
|
||||
"blueberry",
|
||||
"raspberry",
|
||||
"blackberry",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "match nested object array",
|
||||
objects: [][]unstructured.Unstructured{
|
||||
{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "apple",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
map[string]interface{}{
|
||||
"pink": "fuji",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"green": "granny-smith",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"pink": "honeycrisp",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "berry",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
map[string]interface{}{
|
||||
"blue": "blueberry",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"red": "raspberry",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"black": "blackberry",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
map[string]interface{}{
|
||||
"yellow": "cavendish",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"green": "plantain",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: []Filter{
|
||||
{
|
||||
field: []string{"data", "attributes", "green"},
|
||||
match: "plantain",
|
||||
},
|
||||
},
|
||||
want: []unstructured.Unstructured{
|
||||
{
|
||||
Object: map[string]interface{}{
|
||||
"kind": "fruit",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "banana",
|
||||
},
|
||||
"data": map[string]interface{}{
|
||||
"attributes": []interface{}{
|
||||
[]interface{}{
|
||||
map[string]interface{}{
|
||||
"yellow": "cavendish",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"green": "plantain",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ch := make(chan []unstructured.Unstructured)
|
||||
go func() {
|
||||
for _, o := range test.objects {
|
||||
ch <- o
|
||||
}
|
||||
close(ch)
|
||||
}()
|
||||
got := FilterList(ch, test.filters)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -5,11 +5,11 @@ package partition
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strconv"
|
||||
|
||||
"github.com/rancher/apiserver/pkg/types"
|
||||
"github.com/rancher/steve/pkg/stores/partition/listprocessor"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@ -18,8 +18,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
)
|
||||
|
||||
const defaultLimit = 100000
|
||||
|
||||
// Partitioner is an interface for interacting with partitions.
|
||||
type Partitioner interface {
|
||||
Lookup(apiOp *types.APIRequest, schema *types.APISchema, verb, id string) (Partition, error)
|
||||
@ -122,19 +120,18 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
|
||||
Partitions: partitions,
|
||||
}
|
||||
|
||||
resume := apiOp.Request.URL.Query().Get("continue")
|
||||
limit := getLimit(apiOp.Request)
|
||||
opts := listprocessor.ParseQuery(apiOp)
|
||||
|
||||
list, err := lister.List(apiOp.Context(), limit, resume)
|
||||
stream, err := lister.List(apiOp.Context(), opts.ChunkSize, opts.Resume)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
for items := range list {
|
||||
for _, item := range items {
|
||||
item := item
|
||||
result.Objects = append(result.Objects, toAPI(schema, &item))
|
||||
}
|
||||
list := listprocessor.FilterList(stream, opts.Filters)
|
||||
|
||||
for _, item := range list {
|
||||
item := item
|
||||
result.Objects = append(result.Objects, toAPI(schema, &item))
|
||||
}
|
||||
|
||||
result.Revision = lister.Revision()
|
||||
@ -213,21 +210,6 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// getLimit extracts the limit parameter from the request or sets a default of 100000.
|
||||
// Since a default is always set, this implies that clients must always be
|
||||
// aware that the list may be incomplete.
|
||||
func getLimit(req *http.Request) int {
|
||||
limitString := req.URL.Query().Get("limit")
|
||||
limit, err := strconv.Atoi(limitString)
|
||||
if err != nil {
|
||||
limit = 0
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = defaultLimit
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func toAPI(schema *types.APISchema, obj runtime.Object) types.APIObject {
|
||||
if obj == nil || reflect.ValueOf(obj).IsNil() {
|
||||
return types.APIObject{}
|
||||
|
||||
@ -198,6 +198,86 @@ func TestList(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with filters",
|
||||
apiOps: []*types.APIRequest{
|
||||
newRequest("filter=data.color=green"),
|
||||
newRequest("filter=data.color=green&filter=metadata.name=bramley"),
|
||||
},
|
||||
partitions: []Partition{
|
||||
mockPartition{
|
||||
name: "all",
|
||||
},
|
||||
},
|
||||
objects: map[string]*unstructured.UnstructuredList{
|
||||
"all": {
|
||||
Items: []unstructured.Unstructured{
|
||||
newApple("fuji").Unstructured,
|
||||
newApple("granny-smith").Unstructured,
|
||||
newApple("bramley").Unstructured,
|
||||
newApple("crispin").Unstructured,
|
||||
},
|
||||
},
|
||||
},
|
||||
want: []types.APIObjectList{
|
||||
{
|
||||
Objects: []types.APIObject{
|
||||
newApple("granny-smith").toObj(),
|
||||
newApple("bramley").toObj(),
|
||||
},
|
||||
},
|
||||
{
|
||||
Objects: []types.APIObject{
|
||||
newApple("bramley").toObj(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multi-partition with filters",
|
||||
apiOps: []*types.APIRequest{
|
||||
newRequest("filter=data.category=baking"),
|
||||
},
|
||||
partitions: []Partition{
|
||||
mockPartition{
|
||||
name: "pink",
|
||||
},
|
||||
mockPartition{
|
||||
name: "green",
|
||||
},
|
||||
mockPartition{
|
||||
name: "yellow",
|
||||
},
|
||||
},
|
||||
objects: map[string]*unstructured.UnstructuredList{
|
||||
"pink": {
|
||||
Items: []unstructured.Unstructured{
|
||||
newApple("fuji").with(map[string]string{"category": "eating"}).Unstructured,
|
||||
newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).Unstructured,
|
||||
},
|
||||
},
|
||||
"green": {
|
||||
Items: []unstructured.Unstructured{
|
||||
newApple("granny-smith").with(map[string]string{"category": "baking"}).Unstructured,
|
||||
newApple("bramley").with(map[string]string{"category": "eating"}).Unstructured,
|
||||
},
|
||||
},
|
||||
"yellow": {
|
||||
Items: []unstructured.Unstructured{
|
||||
newApple("crispin").with(map[string]string{"category": "baking"}).Unstructured,
|
||||
},
|
||||
},
|
||||
},
|
||||
want: []types.APIObjectList{
|
||||
{
|
||||
Objects: []types.APIObject{
|
||||
newApple("honeycrisp").with(map[string]string{"category": "eating,baking"}).toObj(),
|
||||
newApple("granny-smith").with(map[string]string{"category": "baking"}).toObj(),
|
||||
newApple("crispin").with(map[string]string{"category": "baking"}).toObj(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@ -353,3 +433,10 @@ func (a apple) toObj() types.APIObject {
|
||||
Object: &a.Unstructured,
|
||||
}
|
||||
}
|
||||
|
||||
func (a apple) with(data map[string]string) apple {
|
||||
for k, v := range data {
|
||||
a.Object["data"].(map[string]interface{})[k] = v
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user