diff --git a/go.mod b/go.mod index a6fdc064..e5b88978 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/adrg/xdg v0.5.3 github.com/golang/protobuf v1.5.4 github.com/google/gnostic-models v0.6.9 - github.com/google/go-cmp v0.6.0 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 github.com/pborman/uuid v1.2.1 @@ -80,6 +79,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/cel-go v0.22.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect diff --git a/pkg/sqlcache/Readme.md b/pkg/sqlcache/Readme.md index 99da82a2..58709137 100644 --- a/pkg/sqlcache/Readme.md +++ b/pkg/sqlcache/Readme.md @@ -25,10 +25,13 @@ like any other informer, but with a wider array of options. The options are conf ### List Options ListOptions includes the following: * Match filters for indexed fields. Filters are for specifying the value a given field in an object should be in order to -be included in the list. Filters can be set to equals or not equals. Filters can be set to look for partial matches or -exact (strict) matches. Filters can be OR'd and AND'd with one another. Filters only work on fields that have been indexed. -* Primary field and secondary field sorting order. Can choose up to two fields to sort on. Sort order can be ascending -or descending. Default sorting is to sort on metadata.namespace in ascending first and then sort on metadata.name. +be included in the list. Filters are similar to the operators on labels in the `kubectl` CLI. Filters can be set to look for partial matches or +exact (strict) matches. Filters can be OR'd and AND'd with one another. A query of the form `filter=field1 OP1 val1,field2 OP2 val2` is an `OR` test, +while separate filters are AND'd together, as in `filter=field1 OP1 val1&filter=field2 OP2 val2`. +* Filters only work on fields that have been indexed. All `metadata.labels` are also indexed. +* Any number of sort fields can be specified, but must be comma-separated in a single `sort=....` query. +Precede each field with a dash (`-`) to sort descending. The default sort is `sort=metadata.namespace,metadata.name` +(sort first by namespace, then name). * Page size to specify how many items to include in a response. * Page number to specify offset. For example, a page size of 50 and a page number of 2, will return items starting at index 50. Index will be dependent on sort. Page numbers start at 1. @@ -95,10 +98,12 @@ intended to be used as a way of enforcing RBAC. ## Technical Information ### SQL Tables -There are three tables that are created for the ListOption informer: +There are four tables that are created for the ListOption informer: * object table - this contains objects, including all their fields, as blobs. These blobs may be encrypted. * fields table - this contains specific fields of value for objects. These are specified on informer create and are fields that it is desired to filter or order on. +* labels table - this contains the labels for each object in the object table. +They go in a separate table because an object can have any number of labels. * indices table - the indices table stores indexes created and objects' values for each index. This backs the generic indexer that contains the functionality needed to conform to cache.Indexer. @@ -136,16 +141,12 @@ have the following indexes by default: ### ListOptions Behavior Defaults: -* Sort.PrimaryField: `metadata.namespace` -* Sort.SecondaryField: `metadata.name` -* Sort.PrimaryOrder: `ASC` (ascending) -* Sort.SecondaryOrder: `ASC` (ascending) +* `sort=metadata.namespace,metadata.name` (ascending order for both) * All filters have partial matching set to false by default There are some uncommon ways someone could use ListOptions where it would be difficult to predict what the result would be. Below is a non-exhaustive list of some of these cases and what the behavior is: * Setting Pagination.Page but not Pagination.PageSize will cause Page to be ignored -* Setting Sort.SecondaryField only will sort as though it was Sort.PrimaryField. Sort.SecondaryOrder will still be applied and Sort.PrimaryOrder will be ignored ### Writing Secure Queries diff --git a/pkg/sqlcache/informer/informer.go b/pkg/sqlcache/informer/informer.go index 8ba193f7..dd162662 100644 --- a/pkg/sqlcache/informer/informer.go +++ b/pkg/sqlcache/informer/informer.go @@ -10,6 +10,7 @@ import ( "github.com/rancher/steve/pkg/sqlcache/db" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" sqlStore "github.com/rancher/steve/pkg/sqlcache/store" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -29,7 +30,7 @@ type Informer struct { } type ByOptionsLister interface { - ListByOptions(ctx context.Context, lo ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) + ListByOptions(ctx context.Context, lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) } // this is set to a var so that it can be overridden by test code for mocking purposes @@ -102,7 +103,7 @@ func NewInformer(ctx context.Context, client dynamic.ResourceInterface, fields [ // - the total number of resources (returned list might be a subset depending on pagination options in lo) // - a continue token, if there are more pages after the returned one // - an error instead of all of the above if anything went wrong -func (i *Informer) ListByOptions(ctx context.Context, lo ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) { +func (i *Informer) ListByOptions(ctx context.Context, lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) { return i.ByOptionsLister.ListByOptions(ctx, lo, partitions, namespace) } diff --git a/pkg/sqlcache/informer/informer_mocks_test.go b/pkg/sqlcache/informer/informer_mocks_test.go index 9eff0612..4cde0b67 100644 --- a/pkg/sqlcache/informer/informer_mocks_test.go +++ b/pkg/sqlcache/informer/informer_mocks_test.go @@ -14,6 +14,7 @@ import ( reflect "reflect" partition "github.com/rancher/steve/pkg/sqlcache/partition" + sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes" gomock "go.uber.org/mock/gomock" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -42,7 +43,7 @@ func (m *MockByOptionsLister) EXPECT() *MockByOptionsListerMockRecorder { } // ListByOptions mocks base method. -func (m *MockByOptionsLister) ListByOptions(arg0 context.Context, arg1 ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { +func (m *MockByOptionsLister) ListByOptions(arg0 context.Context, arg1 *sqltypes.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListByOptions", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*unstructured.UnstructuredList) diff --git a/pkg/sqlcache/informer/informer_test.go b/pkg/sqlcache/informer/informer_test.go index 4fe84c6a..b030e1d4 100644 --- a/pkg/sqlcache/informer/informer_test.go +++ b/pkg/sqlcache/informer/informer_test.go @@ -9,6 +9,7 @@ import ( "github.com/rancher/steve/pkg/sqlcache/db" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -313,7 +314,7 @@ func TestInformerListByOptions(t *testing.T) { informer := &Informer{ ByOptionsLister: indexer, } - lo := ListOptions{} + lo := sqltypes.ListOptions{} var partitions []partition.Partition ns := "somens" expectedList := &unstructured.UnstructuredList{ @@ -324,8 +325,8 @@ func TestInformerListByOptions(t *testing.T) { } expectedTotal := len(expectedList.Items) expectedContinueToken := "123" - indexer.EXPECT().ListByOptions(context.Background(), lo, partitions, ns).Return(expectedList, expectedTotal, expectedContinueToken, nil) - list, total, continueToken, err := informer.ListByOptions(context.Background(), lo, partitions, ns) + indexer.EXPECT().ListByOptions(context.Background(), &lo, partitions, ns).Return(expectedList, expectedTotal, expectedContinueToken, nil) + list, total, continueToken, err := informer.ListByOptions(context.Background(), &lo, partitions, ns) assert.Nil(t, err) assert.Equal(t, expectedList, list) assert.Equal(t, len(expectedList.Items), total) @@ -336,11 +337,11 @@ func TestInformerListByOptions(t *testing.T) { informer := &Informer{ ByOptionsLister: indexer, } - lo := ListOptions{} + lo := sqltypes.ListOptions{} var partitions []partition.Partition ns := "somens" - indexer.EXPECT().ListByOptions(context.Background(), lo, partitions, ns).Return(nil, 0, "", fmt.Errorf("error")) - _, _, _, err := informer.ListByOptions(context.Background(), lo, partitions, ns) + indexer.EXPECT().ListByOptions(context.Background(), &lo, partitions, ns).Return(nil, 0, "", fmt.Errorf("error")) + _, _, _, err := informer.ListByOptions(context.Background(), &lo, partitions, ns) assert.NotNil(t, err) }}) t.Parallel() diff --git a/pkg/sqlcache/informer/listoption_indexer.go b/pkg/sqlcache/informer/listoption_indexer.go index b0c57547..9885fec1 100644 --- a/pkg/sqlcache/informer/listoption_indexer.go +++ b/pkg/sqlcache/informer/listoption_indexer.go @@ -7,11 +7,11 @@ import ( "errors" "fmt" "regexp" - "sort" "strconv" "strings" "github.com/rancher/steve/pkg/sqlcache/db/transaction" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/tools/cache" @@ -93,6 +93,7 @@ func NewListOptionIndexer(ctx context.Context, fields [][]string, s Store, names for _, f := range fields { indexedFields = append(indexedFields, toColumnName(f)) } + fmt.Println(indexedFields) l := &ListOptionIndexer{ Indexer: i, @@ -247,11 +248,13 @@ func (l *ListOptionIndexer) deleteLabels(key string, tx transaction.Client) erro // - the total number of resources (returned list might be a subset depending on pagination options in lo) // - a continue token, if there are more pages after the returned one // - an error instead of all of the above if anything went wrong -func (l *ListOptionIndexer) ListByOptions(ctx context.Context, lo ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) { +func (l *ListOptionIndexer) ListByOptions(ctx context.Context, lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) { queryInfo, err := l.constructQuery(lo, partitions, namespace, db.Sanitize(l.GetName())) if err != nil { return nil, 0, "", err } + logrus.Debugf("ListOptionIndexer prepared statement: %v", queryInfo.query) + logrus.Debugf("Params: %v", queryInfo.params) return l.executeQuery(ctx, queryInfo) } @@ -266,212 +269,6 @@ type QueryInfo struct { offset int } -func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) { - ensureSortLabelsAreSelected(&lo) - queryInfo := &QueryInfo{} - queryUsesLabels := hasLabelFilter(lo.Filters) - joinTableIndexByLabelName := make(map[string]int) - - // First, what kind of filtering will we be doing? - // 1- Intro: SELECT and JOIN clauses - // There's a 1:1 correspondence between a base table and its _Fields table - // but it's possible that a key has no associated labels, so if we're doing a - // non-existence test on labels we need to do a LEFT OUTER JOIN - distinctModifier := "" - if queryUsesLabels { - distinctModifier = " DISTINCT" - } - query := fmt.Sprintf(`SELECT%s o.object, o.objectnonce, o.dekid FROM "%s" o`, distinctModifier, dbName) - query += "\n " - query += fmt.Sprintf(`JOIN "%s_fields" f ON o.key = f.key`, dbName) - if queryUsesLabels { - for i, orFilter := range lo.Filters { - for j, filter := range orFilter.Filters { - if isLabelFilter(&filter) { - labelName := filter.Field[2] - _, ok := joinTableIndexByLabelName[labelName] - if !ok { - // Make the lt index 1-based for readability - jtIndex := i + j + 1 - joinTableIndexByLabelName[labelName] = jtIndex - query += "\n " - query += fmt.Sprintf(`LEFT OUTER JOIN "%s_labels" lt%d ON o.key = lt%d.key`, dbName, jtIndex, jtIndex) - } - } - } - } - } - params := []any{} - - // 2- Filtering: WHERE clauses (from lo.Filters) - whereClauses := []string{} - for _, orFilters := range lo.Filters { - orClause, orParams, err := l.buildORClauseFromFilters(orFilters, dbName, joinTableIndexByLabelName) - if err != nil { - return queryInfo, err - } - if orClause == "" { - continue - } - whereClauses = append(whereClauses, orClause) - params = append(params, orParams...) - } - - // WHERE clauses (from namespace) - if namespace != "" && namespace != "*" { - whereClauses = append(whereClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) - params = append(params, namespace) - } - - // WHERE clauses (from partitions and their corresponding parameters) - partitionClauses := []string{} - for _, thisPartition := range partitions { - if thisPartition.Passthrough { - // nothing to do, no extra filtering to apply by definition - } else { - singlePartitionClauses := []string{} - - // filter by namespace - if thisPartition.Namespace != "" && thisPartition.Namespace != "*" { - singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) - params = append(params, thisPartition.Namespace) - } - - // optionally filter by names - if !thisPartition.All { - names := thisPartition.Names - - if len(names) == 0 { - // degenerate case, there will be no results - singlePartitionClauses = append(singlePartitionClauses, "FALSE") - } else { - singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.name" IN (?%s)`, strings.Repeat(", ?", len(thisPartition.Names)-1))) - // sort for reproducibility - sortedNames := thisPartition.Names.UnsortedList() - sort.Strings(sortedNames) - for _, name := range sortedNames { - params = append(params, name) - } - } - } - - if len(singlePartitionClauses) > 0 { - partitionClauses = append(partitionClauses, strings.Join(singlePartitionClauses, " AND ")) - } - } - } - if len(partitions) == 0 { - // degenerate case, there will be no results - whereClauses = append(whereClauses, "FALSE") - } - if len(partitionClauses) == 1 { - whereClauses = append(whereClauses, partitionClauses[0]) - } - if len(partitionClauses) > 1 { - whereClauses = append(whereClauses, "(\n ("+strings.Join(partitionClauses, ") OR\n (")+")\n)") - } - - if len(whereClauses) > 0 { - query += "\n WHERE\n " - for index, clause := range whereClauses { - query += fmt.Sprintf("(%s)", clause) - if index == len(whereClauses)-1 { - break - } - query += " AND\n " - } - } - - // before proceeding, save a copy of the query and params without LIMIT/OFFSET/ORDER info - // for COUNTing all results later - countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", query) - countParams := params[:] - - // 3- Sorting: ORDER BY clauses (from lo.Sort) - if len(lo.Sort.Fields) != len(lo.Sort.Orders) { - return nil, fmt.Errorf("sort fields length %d != sort orders length %d", len(lo.Sort.Fields), len(lo.Sort.Orders)) - } - if len(lo.Sort.Fields) > 0 { - orderByClauses := []string{} - for i, field := range lo.Sort.Fields { - if isLabelsFieldList(field) { - clause, sortParam, err := buildSortLabelsClause(field[2], joinTableIndexByLabelName, lo.Sort.Orders[i] == ASC) - if err != nil { - return nil, err - } - orderByClauses = append(orderByClauses, clause) - params = append(params, sortParam) - } else { - columnName := toColumnName(field) - if err := l.validateColumn(columnName); err != nil { - return queryInfo, err - } - direction := "ASC" - if lo.Sort.Orders[i] == DESC { - direction = "DESC" - } - orderByClauses = append(orderByClauses, fmt.Sprintf(`f."%s" %s`, columnName, direction)) - } - } - query += "\n ORDER BY " - query += strings.Join(orderByClauses, ", ") - } else { - // make sure one default order is always picked - if l.namespaced { - query += "\n ORDER BY f.\"metadata.namespace\" ASC, f.\"metadata.name\" ASC " - } else { - query += "\n ORDER BY f.\"metadata.name\" ASC " - } - } - - // 4- Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume) - - limitClause := "" - // take the smallest limit between lo.Pagination and lo.ChunkSize - limit := lo.Pagination.PageSize - if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) { - limit = lo.ChunkSize - } - if limit > 0 { - limitClause = "\n LIMIT ?" - params = append(params, limit) - } - - // OFFSET clause (from lo.Pagination and/or lo.Resume) - offsetClause := "" - offset := 0 - if lo.Resume != "" { - offsetInt, err := strconv.Atoi(lo.Resume) - if err != nil { - return queryInfo, err - } - offset = offsetInt - } - if lo.Pagination.Page >= 1 { - offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1) - } - if offset > 0 { - offsetClause = "\n OFFSET ?" - params = append(params, offset) - } - if limit > 0 || offset > 0 { - query += limitClause - query += offsetClause - queryInfo.countQuery = countQuery - queryInfo.countParams = countParams - queryInfo.limit = limit - queryInfo.offset = offset - } - // Otherwise leave these as default values and the executor won't do pagination work - - logrus.Debugf("ListOptionIndexer prepared statement: %v", query) - logrus.Debugf("Params: %v", params) - queryInfo.query = query - queryInfo.params = params - - return queryInfo, nil -} - func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryInfo) (result *unstructured.UnstructuredList, total int, token string, err error) { stmt := l.Prepare(queryInfo.query) defer func() { @@ -529,337 +326,12 @@ func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryIn return toUnstructuredList(items), total, continueToken, nil } -func (l *ListOptionIndexer) validateColumn(column string) error { - for _, v := range l.indexedFields { - if v == column { - return nil - } +func extractSubFields(fields string) []string { + subfields := make([]string, 0) + for _, subField := range subfieldRegex.FindAllString(fields, -1) { + subfields = append(subfields, strings.TrimSuffix(subField, ".")) } - return fmt.Errorf("column is invalid [%s]: %w", column, ErrInvalidColumn) -} - -// buildORClause creates an SQLite compatible query that ORs conditions built from passed filters -func (l *ListOptionIndexer) buildORClauseFromFilters(orFilters OrFilter, dbName string, joinTableIndexByLabelName map[string]int) (string, []any, error) { - var params []any - clauses := make([]string, 0, len(orFilters.Filters)) - var newParams []any - var newClause string - var err error - - for _, filter := range orFilters.Filters { - if isLabelFilter(&filter) { - index, ok := joinTableIndexByLabelName[filter.Field[2]] - if !ok { - return "", nil, fmt.Errorf("internal error: no index for label name %s", filter.Field[2]) - } - newClause, newParams, err = l.getLabelFilter(index, filter, dbName) - } else { - newClause, newParams, err = l.getFieldFilter(filter) - } - if err != nil { - return "", nil, err - } - clauses = append(clauses, newClause) - params = append(params, newParams...) - } - switch len(clauses) { - case 0: - return "", params, nil - case 1: - return clauses[0], params, nil - } - return fmt.Sprintf("(%s)", strings.Join(clauses, ") OR (")), params, nil -} - -func buildSortLabelsClause(labelName string, joinTableIndexByLabelName map[string]int, isAsc bool) (string, string, error) { - ltIndex, ok := joinTableIndexByLabelName[labelName] - if !ok { - return "", "", fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) - } - stmt := fmt.Sprintf(`CASE lt%d.label WHEN ? THEN lt%d.value ELSE NULL END`, ltIndex, ltIndex) - dir := "ASC" - nullsPosition := "LAST" - if !isAsc { - dir = "DESC" - nullsPosition = "FIRST" - } - return fmt.Sprintf("(%s) %s NULLS %s", stmt, dir, nullsPosition), labelName, nil -} - -// If the user tries to sort on a particular label without mentioning it in a query, -// it turns out that the sort-directive is ignored. It could be that the sqlite engine -// is doing some kind of optimization on the `select distinct`, but verifying an otherwise -// unreferenced label exists solves this problem. -// And it's better to do this by modifying the ListOptions object. -// There are no thread-safety issues in doing this because the ListOptions object is -// created in Store.ListByPartitions, and that ends up calling ListOptionIndexer.ConstructQuery. -// No other goroutines access this object. -func ensureSortLabelsAreSelected(lo *ListOptions) { - if len(lo.Sort.Fields) == 0 { - return - } - unboundSortLabels := make(map[string]bool) - for _, fieldList := range lo.Sort.Fields { - if isLabelsFieldList(fieldList) { - unboundSortLabels[fieldList[2]] = true - } - } - if len(unboundSortLabels) == 0 { - return - } - // If we have sort directives but no filters, add an exists-filter for each label. - if lo.Filters == nil || len(lo.Filters) == 0 { - lo.Filters = make([]OrFilter, 1) - lo.Filters[0].Filters = make([]Filter, len(unboundSortLabels)) - i := 0 - for labelName := range unboundSortLabels { - lo.Filters[0].Filters[i] = Filter{ - Field: []string{"metadata", "labels", labelName}, - Op: Exists, - } - i++ - } - return - } - // The gotcha is we have to bind the labels for each set of orFilters, so copy them each time - for i, orFilters := range lo.Filters { - copyUnboundSortLabels := make(map[string]bool, len(unboundSortLabels)) - for k, v := range unboundSortLabels { - copyUnboundSortLabels[k] = v - } - for _, filter := range orFilters.Filters { - if isLabelFilter(&filter) { - copyUnboundSortLabels[filter.Field[2]] = false - } - } - // Now for any labels that are still true, add another where clause - for labelName, needsBinding := range copyUnboundSortLabels { - if needsBinding { - // `orFilters` is a copy of lo.Filters[i], so reference the original. - lo.Filters[i].Filters = append(lo.Filters[i].Filters, Filter{ - Field: []string{"metadata", "labels", labelName}, - Op: Exists, - }) - } - } - } -} - -// Possible ops from the k8s parser: -// KEY = and == (same) VALUE -// KEY != VALUE -// KEY exists [] # ,KEY, => this filter -// KEY ! [] # ,!KEY, => assert KEY doesn't exist -// KEY in VALUES -// KEY notin VALUES - -func (l *ListOptionIndexer) getFieldFilter(filter Filter) (string, []any, error) { - opString := "" - escapeString := "" - columnName := toColumnName(filter.Field) - if err := l.validateColumn(columnName); err != nil { - return "", nil, err - } - switch filter.Op { - case Eq: - if filter.Partial { - opString = "LIKE" - escapeString = escapeBackslashDirective - } else { - opString = "=" - } - clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) - return clause, []any{formatMatchTarget(filter)}, nil - case NotEq: - if filter.Partial { - opString = "NOT LIKE" - escapeString = escapeBackslashDirective - } else { - opString = "!=" - } - clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) - return clause, []any{formatMatchTarget(filter)}, nil - - case Lt, Gt: - sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`f."%s" %s ?`, columnName, sym) - return clause, []any{target}, nil - - case Exists, NotExists: - return "", nil, errors.New("NULL and NOT NULL tests aren't supported for non-label queries") - - case In: - fallthrough - case NotIn: - target := "()" - if len(filter.Matches) > 0 { - target = fmt.Sprintf("(?%s)", strings.Repeat(", ?", len(filter.Matches)-1)) - } - opString = "IN" - if filter.Op == NotIn { - opString = "NOT IN" - } - clause := fmt.Sprintf(`f."%s" %s %s`, columnName, opString, target) - matches := make([]any, len(filter.Matches)) - for i, match := range filter.Matches { - matches[i] = match - } - return clause, matches, nil - } - - return "", nil, fmt.Errorf("unrecognized operator: %s", opString) -} - -func (l *ListOptionIndexer) getLabelFilter(index int, filter Filter, dbName string) (string, []any, error) { - opString := "" - escapeString := "" - matchFmtToUse := strictMatchFmt - labelName := filter.Field[2] - switch filter.Op { - case Eq: - if filter.Partial { - opString = "LIKE" - escapeString = escapeBackslashDirective - matchFmtToUse = matchFmt - } else { - opString = "=" - } - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?%s`, index, index, opString, escapeString) - return clause, []any{labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil - - case NotEq: - if filter.Partial { - opString = "NOT LIKE" - escapeString = escapeBackslashDirective - matchFmtToUse = matchFmt - } else { - opString = "!=" - } - subFilter := Filter{ - Field: filter.Field, - Op: NotExists, - } - existenceClause, subParams, err := l.getLabelFilter(index, subFilter, dbName) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value %s ?%s)`, existenceClause, index, index, opString, escapeString) - params := append(subParams, labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) - return clause, params, nil - - case Lt, Gt: - sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?`, index, index, sym) - return clause, []any{labelName, target}, nil - - case Exists: - clause := fmt.Sprintf(`lt%d.label = ?`, index) - return clause, []any{labelName}, nil - - case NotExists: - clause := fmt.Sprintf(`o.key NOT IN (SELECT o1.key FROM "%s" o1 - JOIN "%s_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "%s_labels" lt%di1 ON o1.key = lt%di1.key - WHERE lt%di1.label = ?)`, dbName, dbName, dbName, index, index, index) - return clause, []any{labelName}, nil - - case In: - target := "(?" - if len(filter.Matches) > 0 { - target += strings.Repeat(", ?", len(filter.Matches)-1) - } - target += ")" - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value IN %s`, index, index, target) - matches := make([]any, len(filter.Matches)+1) - matches[0] = labelName - for i, match := range filter.Matches { - matches[i+1] = match - } - return clause, matches, nil - - case NotIn: - target := "(?" - if len(filter.Matches) > 0 { - target += strings.Repeat(", ?", len(filter.Matches)-1) - } - target += ")" - subFilter := Filter{ - Field: filter.Field, - Op: NotExists, - } - existenceClause, subParams, err := l.getLabelFilter(index, subFilter, dbName) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value NOT IN %s)`, existenceClause, index, index, target) - matches := append(subParams, labelName) - for _, match := range filter.Matches { - matches = append(matches, match) - } - return clause, matches, nil - } - return "", nil, fmt.Errorf("unrecognized operator: %s", opString) -} - -func prepareComparisonParameters(op Op, target string) (string, float64, error) { - num, err := strconv.ParseFloat(target, 32) - if err != nil { - return "", 0, err - } - switch op { - case Lt: - return "<", num, nil - case Gt: - return ">", num, nil - } - return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op) -} - -func formatMatchTarget(filter Filter) string { - format := strictMatchFmt - if filter.Partial { - format = matchFmt - } - return formatMatchTargetWithFormatter(filter.Matches[0], format) -} - -func formatMatchTargetWithFormatter(match string, format string) string { - // To allow matches on the backslash itself, the character needs to be replaced first. - // Otherwise, it will undo the following replacements. - match = strings.ReplaceAll(match, `\`, `\\`) - match = strings.ReplaceAll(match, `_`, `\_`) - match = strings.ReplaceAll(match, `%`, `\%`) - return fmt.Sprintf(format, match) -} - -// There are two kinds of string arrays to turn into a string, based on the last value in the array -// simple: ["a", "b", "conformsToIdentifier"] => "a.b.conformsToIdentifier" -// complex: ["a", "b", "foo.io/stuff"] => "a.b[foo.io/stuff]" - -func smartJoin(s []string) string { - if len(s) == 0 { - return "" - } - if len(s) == 1 { - return s[0] - } - lastBit := s[len(s)-1] - simpleName := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) - if simpleName.MatchString(lastBit) { - return strings.Join(s, ".") - } - return fmt.Sprintf("%s[%s]", strings.Join(s[0:len(s)-1], "."), lastBit) -} - -// toColumnName returns the column name corresponding to a field expressed as string slice -func toColumnName(s []string) string { - return db.Sanitize(smartJoin(s)) + return subfields } // getField extracts the value of a field expressed as a string path from an unstructured object @@ -920,31 +392,9 @@ func getField(a any, field string) (any, error) { return obj, nil } -func extractSubFields(fields string) []string { - subfields := make([]string, 0) - for _, subField := range subfieldRegex.FindAllString(fields, -1) { - subfields = append(subfields, strings.TrimSuffix(subField, ".")) - } - return subfields -} - -func isLabelFilter(f *Filter) bool { - return len(f.Field) >= 2 && f.Field[0] == "metadata" && f.Field[1] == "labels" -} - -func hasLabelFilter(filters []OrFilter) bool { - for _, outerFilter := range filters { - for _, filter := range outerFilter.Filters { - if isLabelFilter(&filter) { - return true - } - } - } - return false -} - -func isLabelsFieldList(fields []string) bool { - return len(fields) == 3 && fields[0] == "metadata" && fields[1] == "labels" +// toColumnName returns the column name corresponding to a field expressed as string slice +func toColumnName(s []string) string { + return db.Sanitize(smartJoin(s)) } // toUnstructuredList turns a slice of unstructured objects into an unstructured.UnstructuredList diff --git a/pkg/sqlcache/informer/listoption_indexer_test.go b/pkg/sqlcache/informer/listoption_indexer_test.go index 4d7c397c..39f226ad 100644 --- a/pkg/sqlcache/informer/listoption_indexer_test.go +++ b/pkg/sqlcache/informer/listoption_indexer_test.go @@ -9,19 +9,12 @@ package informer import ( "context" "database/sql" - "errors" "fmt" - "reflect" - "strings" "testing" "github.com/rancher/steve/pkg/sqlcache/db" - "github.com/rancher/steve/pkg/sqlcache/partition" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" ) func TestNewListOptionIndexer(t *testing.T) { @@ -251,1540 +244,3 @@ func TestNewListOptionIndexer(t *testing.T) { t.Run(test.description, func(t *testing.T) { test.test(t) }) } } - -func TestListByOptions(t *testing.T) { - type testCase struct { - description string - listOptions ListOptions - partitions []partition.Partition - ns string - expectedCountStmt string - expectedCountStmtArgs []any - expectedStmt string - expectedStmtArgs []any - extraIndexedFields []string - expectedList *unstructured.UnstructuredList - returnList []any - expectedContToken string - expectedErr error - } - - testObject := testStoreObject{Id: "something", Val: "a"} - unstrTestObjectMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testObject) - assert.Nil(t, err) - - var tests []testCase - tests = append(tests, testCase{ - description: "ListByOptions() with no errors returned, should not return an error", - listOptions: ListOptions{}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions() with an empty filter, should not return an error", - listOptions: ListOptions{ - Filters: []OrFilter{{[]Filter{}}}, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt", - listOptions: ListOptions{ChunkSize: 2}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ?`, - expectedStmtArgs: []interface{}{2}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Resume set should set offset in prepared sql.Stmt", - listOptions: ListOptions{Resume: "4"}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - OFFSET ?`, - expectedStmtArgs: []interface{}{4}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter with Op set top NotEq should select where that filter is not true in prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" NOT LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter with Partial set to true should select where that partial match on that filter's value is true in prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with multiple filters should select where any of those filters are true in prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: true, - }, - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"someothervalue"}, - Op: Eq, - Partial: true, - }, - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somethirdvalue"}, - Op: NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - ((f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%", "%someothervalue%", "%somethirdvalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with multiple OrFilters set should select where all OrFilters contain one filter that is true in prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"value1"}, - Op: Eq, - Partial: false, - }, - { - Field: []string{"status", "someotherfield"}, - Matches: []string{"value2"}, - Op: NotEq, - Partial: false, - }, - }, - }, - { - Filters: []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"value3"}, - Op: Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test4", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - ((f."metadata.somefield" = ?) OR (f."status.someotherfield" != ?)) AND - (f."metadata.somefield" = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"value1", "value2", "value3", "test4"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with labels filter should select the label in the prepared sql.Stmt", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "guard.cattle.io"}, - Matches: []string{"lodgepole"}, - Op: Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test41", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"guard.cattle.io", "%lodgepole%", "test41"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with two labels filters should use a self-join", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "cows"}, - Matches: []string{"milk"}, - Op: Eq, - Partial: false, - }, - }, - }, - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "horses"}, - Matches: []string{"saddles"}, - Op: Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test42", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (lt2.label = ? AND lt2.value = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"cows", "milk", "horses", "saddles", "test42"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with a mix of one label and one non-label query can still self-join", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "cows"}, - Matches: []string{"butter"}, - Op: Eq, - Partial: false, - }, - }, - }, - { - Filters: []Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"wheat"}, - Op: Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test43", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (f."metadata.somefield" = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"cows", "butter", "wheat", "test43"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with only one Sort.Field set should sort on that field only, in ascending order in prepared sql.Stmt", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}}, - Orders: []SortOrder{ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "test5", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.somefield" ASC`, - expectedStmtArgs: []any{"test5"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "sort one field descending", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}}, - Orders: []SortOrder{DESC}, - }, - }, - partitions: []partition.Partition{}, - ns: "test5a", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.somefield" DESC`, - expectedStmtArgs: []any{"test5a"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "sort one unbound label descending", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "labels", "flip"}}, - Orders: []SortOrder{DESC}, - }, - }, - partitions: []partition.Partition{}, - ns: "test5a", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) DESC NULLS FIRST`, - expectedStmtArgs: []any{"flip", "test5a", "flip"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions sorting on two complex fields should sort on the first field in ascending order first and then sort on the second labels field in ascending order in prepared sql.Stmt", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "fields", "3"}, {"metadata", "labels", "stub.io/candy"}}, - Orders: []SortOrder{ASC, ASC}, - }, - }, - extraIndexedFields: []string{"metadata.fields[3]", "metadata.labels[stub.io/candy]"}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY f."metadata.fields[3]" ASC, (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, - expectedStmtArgs: []any{"stub.io/candy", "stub.io/candy"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions sorting on two fields should sort on the first field in ascending order first and then sort on the second field in ascending order in prepared sql.Stmt", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}, {"status", "someotherfield"}}, - Orders: []SortOrder{ASC, ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.somefield" ASC, f."status.someotherfield" ASC`, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions sorting on two fields should sort on the first field in descending order first and then sort on the second field in ascending order in prepared sql.Stmt", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}, {"status", "someotherfield"}}, - Orders: []SortOrder{DESC, ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.somefield" DESC, f."status.someotherfield" ASC`, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions sorting when # fields != # sort orders should return an error", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}, {"status", "someotherfield"}}, - Orders: []SortOrder{DESC, ASC, ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.somefield" DESC, f."status.someotherfield" ASC`, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: fmt.Errorf("sort fields length 2 != sort orders length 3"), - }) - - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.PageSize set should set limit to PageSize in prepared sql.Stmt", - listOptions: ListOptions{ - Pagination: Pagination{ - PageSize: 10, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ?`, - expectedStmtArgs: []any{10}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.Page and no PageSize set should not add anything to prepared sql.Stmt", - listOptions: ListOptions{ - Pagination: Pagination{ - Page: 2, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.Page and PageSize set limit to PageSize and offset to PageSize * (Page - 1) in prepared sql.Stmt", - listOptions: ListOptions{ - Pagination: Pagination{ - PageSize: 10, - Page: 2, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ? - OFFSET ?`, - expectedStmtArgs: []any{10, 10}, - - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Namespace Partition should select only items where metadata.namespace is equal to Namespace and all other conditions are met in prepared sql.Stmt", - partitions: []partition.Partition{ - { - Namespace: "somens", - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ? AND FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somens"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a All Partition should select all items that meet all other conditions in prepared sql.Stmt", - partitions: []partition.Partition{ - { - All: true, - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Passthrough Partition should select all items that meet all other conditions prepared sql.Stmt", - partitions: []partition.Partition{ - { - Passthrough: true, - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Names Partition should select only items where metadata.name equals an items in Names and all other conditions are met in prepared sql.Stmt", - partitions: []partition.Partition{ - { - Names: sets.New[string]("someid", "someotherid"), - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.name" IN (?, ?)) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"someid", "someotherid"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - txClient := NewMockTXClient(gomock.NewController(t)) - store := NewMockStore(gomock.NewController(t)) - stmts := NewMockStmt(gomock.NewController(t)) - i := &Indexer{ - Store: store, - } - lii := &ListOptionIndexer{ - Indexer: i, - indexedFields: []string{"metadata.somefield", "status.someotherfield"}, - } - if len(test.extraIndexedFields) > 0 { - lii.indexedFields = append(lii.indexedFields, test.extraIndexedFields...) - } - queryInfo, err := lii.constructQuery(test.listOptions, test.partitions, test.ns, "something") - if test.expectedErr != nil { - assert.Equal(t, test.expectedErr, err) - return - } - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, queryInfo.query) - if test.expectedStmtArgs == nil { - test.expectedStmtArgs = []any{} - } - assert.Equal(t, test.expectedStmtArgs, queryInfo.params) - assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) - assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) - - stmt := &sql.Stmt{} - rows := &sql.Rows{} - objType := reflect.TypeOf(testObject) - txClient.EXPECT().Stmt(gomock.Any()).Return(stmts).AnyTimes() - store.EXPECT().Prepare(test.expectedStmt).Do(func(a ...any) { - fmt.Println(a) - }).Return(stmt) - if args := test.expectedStmtArgs; args != nil { - stmts.EXPECT().QueryContext(gomock.Any(), gomock.Any()).Return(rows, nil).AnyTimes() - } else if strings.Contains(test.expectedStmt, "LIMIT") { - stmts.EXPECT().QueryContext(gomock.Any(), args...).Return(rows, nil) - txClient.EXPECT().Stmt(gomock.Any()).Return(stmts) - stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) - } else { - stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) - } - store.EXPECT().GetType().Return(objType) - store.EXPECT().GetShouldEncrypt().Return(false) - store.EXPECT().ReadObjects(rows, objType, false).Return(test.returnList, nil) - store.EXPECT().CloseStmt(stmt).Return(nil) - - store.EXPECT().WithTransaction(gomock.Any(), false, gomock.Any()).Return(nil).Do( - func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { - err := f(txClient) - if test.expectedErr == nil { - assert.Nil(t, err) - } else { - assert.Equal(t, test.expectedErr, err) - } - }) - - if test.expectedCountStmt != "" { - store.EXPECT().Prepare(test.expectedCountStmt).Return(stmt) - store.EXPECT().ReadInt(rows).Return(len(test.expectedList.Items), nil) - store.EXPECT().CloseStmt(stmt).Return(nil) - } - list, total, contToken, err := lii.executeQuery(context.Background(), queryInfo) - if test.expectedErr == nil { - assert.Nil(t, err) - } else { - assert.Equal(t, test.expectedErr, err) - } - assert.Equal(t, test.expectedList, list) - assert.Equal(t, len(test.expectedList.Items), total) - assert.Equal(t, test.expectedContToken, contToken) - }) - } -} - -func TestConstructQuery(t *testing.T) { - type testCase struct { - description string - listOptions ListOptions - partitions []partition.Partition - ns string - expectedCountStmt string - expectedCountStmtArgs []any - expectedStmt string - expectedStmtArgs []any - expectedErr error - } - - var tests []testCase - tests = append(tests, testCase{ - description: "TestConstructQuery: handles IN statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"somevalue"}, - Op: In, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.queryField1" IN (?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOT-IN statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"somevalue"}, - Op: NotIn, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.queryField1" NOT IN (?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles EXISTS statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "queryField1"}, - Op: Exists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOT-EXISTS statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "queryField1"}, - Op: NotExists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelEqualFull"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEqualFull", "somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements, match partial", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelEqualPartial"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles != statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelNotEqualFull"}, - Matches: []string{"somevalue"}, - Op: NotEq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNotEqualFull", "labelNotEqualFull", "somevalue"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles != statements for label statements, match partial", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelNotEqualPartial"}, - Matches: []string{"somevalue"}, - Op: NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNotEqualPartial", "labelNotEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles multiple != statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "notEqual1"}, - Matches: []string{"value1"}, - Op: NotEq, - Partial: false, - }, - }, - }, - { - []Filter{ - { - Field: []string{"metadata", "labels", "notEqual2"}, - Matches: []string{"value2"}, - Op: NotEq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt2i1 ON o1.key = lt2i1.key - WHERE lt2i1.label = ?)) OR (lt2.label = ? AND lt2.value != ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"notEqual1", "notEqual1", "value1", "notEqual2", "notEqual2", "value2"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles IN statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelIN"}, - Matches: []string{"somevalue1", "someValue2"}, - Op: In, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value IN (?, ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelIN", "somevalue1", "someValue2"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOTIN statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelNOTIN"}, - Matches: []string{"somevalue1", "someValue2"}, - Op: NotIn, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT IN (?, ?))) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNOTIN", "labelNOTIN", "somevalue1", "someValue2"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles EXISTS statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelEXISTS"}, - Matches: []string{}, - Op: Exists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEXISTS"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOTEXISTS statements for label statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelNOTEXISTS"}, - Matches: []string{}, - Op: NotExists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNOTEXISTS"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles LessThan statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "numericThing"}, - Matches: []string{"5"}, - Op: Lt, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value < ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"numericThing", float64(5)}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles GreaterThan statements", - listOptions: ListOptions{Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "numericThing"}, - Matches: []string{"35"}, - Op: Gt, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value > ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"numericThing", float64(35)}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "multiple filters with a positive label test and a negative non-label test still outer-join", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "junta"}, - Matches: []string{"esther"}, - Op: Eq, - Partial: true, - }, - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"golgi"}, - Op: NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"junta", "%esther%", "%golgi%"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "multiple filters and or-filters with a positive label test and a negative non-label test still outer-join and have correct AND/ORs", - listOptions: ListOptions{Filters: []OrFilter{ - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "nectar"}, - Matches: []string{"stash"}, - Op: Eq, - Partial: true, - }, - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"landlady"}, - Op: NotEq, - Partial: false, - }, - }, - }, - { - Filters: []Filter{ - { - Field: []string{"metadata", "labels", "lawn"}, - Matches: []string{"reba", "coil"}, - Op: In, - }, - { - Field: []string{"metadata", "queryField1"}, - Op: Gt, - Matches: []string{"2"}, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" != ?)) AND - ((lt2.label = ? AND lt2.value IN (?, ?)) OR (f."metadata.queryField1" > ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"nectar", "%stash%", "landlady", "lawn", "reba", "coil", float64(2)}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements, match partial, sort on metadata.queryField1", - listOptions: ListOptions{ - Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "labels", "labelEqualPartial"}, - Matches: []string{"somevalue"}, - Op: Eq, - Partial: true, - }, - }, - }, - }, - Sort: Sort{ - Fields: [][]string{{"metadata", "queryField1"}}, - Orders: []SortOrder{ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.queryField1" ASC`, - expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ConstructQuery: sorting when # fields < # sort orders should return an error", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}, {"status", "someotherfield"}}, - Orders: []SortOrder{DESC, ASC, ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: "", - expectedStmtArgs: []any{}, - expectedErr: fmt.Errorf("sort fields length 2 != sort orders length 3"), - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: sort on label statements with no query", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "labels", "this"}}, - Orders: []SortOrder{ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, - expectedStmtArgs: []any{"this", "this"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: sort and query on both labels and non-labels without overlap", - listOptions: ListOptions{ - Filters: []OrFilter{ - { - []Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"toys"}, - Op: Eq, - }, - { - Field: []string{"metadata", "labels", "jamb"}, - Matches: []string{"juice"}, - Op: Eq, - }, - }, - }, - }, - Sort: Sort{ - Fields: [][]string{{"metadata", "labels", "this"}, {"status", "queryField2"}}, - Orders: []SortOrder{ASC, DESC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - LEFT OUTER JOIN "something_labels" lt3 ON o.key = lt3.key - WHERE - ((f."metadata.queryField1" = ?) OR (lt2.label = ? AND lt2.value = ?) OR (lt3.label = ?)) AND - (FALSE) - ORDER BY (CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST, f."status.queryField2" DESC`, - expectedStmtArgs: []any{"toys", "jamb", "juice", "this", "this"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ConstructQuery: sorting when # fields > # sort orders should return an error", - listOptions: ListOptions{ - Sort: Sort{ - Fields: [][]string{{"metadata", "somefield"}, {"status", "someotherfield"}, {"metadata", "labels", "a1"}, {"metadata", "labels", "a2"}}, - Orders: []SortOrder{DESC, ASC, ASC}, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: "", - expectedStmtArgs: []any{}, - expectedErr: fmt.Errorf("sort fields length 4 != sort orders length 3"), - }) - - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - store := NewMockStore(gomock.NewController(t)) - i := &Indexer{ - Store: store, - } - lii := &ListOptionIndexer{ - Indexer: i, - indexedFields: []string{"metadata.queryField1", "status.queryField2"}, - } - queryInfo, err := lii.constructQuery(test.listOptions, test.partitions, test.ns, "something") - if test.expectedErr != nil { - assert.Equal(t, test.expectedErr, err) - return - } - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, queryInfo.query) - assert.Equal(t, test.expectedStmtArgs, queryInfo.params) - assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) - assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) - }) - } -} - -func TestSmartJoin(t *testing.T) { - type testCase struct { - description string - fieldArray []string - expectedFieldName string - } - - var tests []testCase - tests = append(tests, testCase{ - description: "single-letter names should be dotted", - fieldArray: []string{"metadata", "labels", "a"}, - expectedFieldName: "metadata.labels.a", - }) - tests = append(tests, testCase{ - description: "underscore should be dotted", - fieldArray: []string{"metadata", "labels", "_"}, - expectedFieldName: "metadata.labels._", - }) - tests = append(tests, testCase{ - description: "simple names should be dotted", - fieldArray: []string{"metadata", "labels", "queryField2"}, - expectedFieldName: "metadata.labels.queryField2", - }) - tests = append(tests, testCase{ - description: "a numeric field should be bracketed", - fieldArray: []string{"metadata", "fields", "43"}, - expectedFieldName: "metadata.fields[43]", - }) - tests = append(tests, testCase{ - description: "a field starting with a number should be bracketed", - fieldArray: []string{"metadata", "fields", "46days"}, - expectedFieldName: "metadata.fields[46days]", - }) - tests = append(tests, testCase{ - description: "compound names should be bracketed", - fieldArray: []string{"metadata", "labels", "rancher.cattle.io/moo"}, - expectedFieldName: "metadata.labels[rancher.cattle.io/moo]", - }) - tests = append(tests, testCase{ - description: "space-separated names should be bracketed", - fieldArray: []string{"metadata", "labels", "space here"}, - expectedFieldName: "metadata.labels[space here]", - }) - tests = append(tests, testCase{ - description: "already-bracketed terms cause double-bracketing and should never be used", - fieldArray: []string{"metadata", "labels[k8s.io/deepcode]"}, - expectedFieldName: "metadata[labels[k8s.io/deepcode]]", - }) - tests = append(tests, testCase{ - description: "an empty array should be an empty string", - fieldArray: []string{}, - expectedFieldName: "", - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - gotFieldName := smartJoin(test.fieldArray) - assert.Equal(t, test.expectedFieldName, gotFieldName) - }) - } -} - -func TestBuildSortLabelsClause(t *testing.T) { - type testCase struct { - description string - labelName string - joinTableIndexByLabelName map[string]int - direction bool - expectedStmt string - expectedParam string - expectedErr string - } - - var tests []testCase - tests = append(tests, testCase{ - description: "TestBuildSortClause: empty index list errors", - labelName: "emptyListError", - expectedErr: `internal error: no join-table index given for labelName "emptyListError"`, - }) - tests = append(tests, testCase{ - description: "TestBuildSortClause: hit ascending", - labelName: "testBSL1", - joinTableIndexByLabelName: map[string]int{"testBSL1": 3}, - direction: true, - expectedStmt: `(CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST`, - expectedParam: "testBSL1", - }) - tests = append(tests, testCase{ - description: "TestBuildSortClause: hit descending", - labelName: "testBSL2", - joinTableIndexByLabelName: map[string]int{"testBSL2": 4}, - direction: false, - expectedStmt: `(CASE lt4.label WHEN ? THEN lt4.value ELSE NULL END) DESC NULLS FIRST`, - expectedParam: "testBSL2", - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - stmt, param, err := buildSortLabelsClause(test.labelName, test.joinTableIndexByLabelName, test.direction) - if test.expectedErr != "" { - assert.Equal(t, test.expectedErr, err.Error()) - } else { - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, stmt) - assert.Equal(t, test.expectedParam, param) - } - }) - } -} diff --git a/pkg/sqlcache/informer/query_generator.go b/pkg/sqlcache/informer/query_generator.go new file mode 100644 index 00000000..22dd77ab --- /dev/null +++ b/pkg/sqlcache/informer/query_generator.go @@ -0,0 +1,1102 @@ +package informer + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" +) + +var ( + badTableNameChars = regexp.MustCompile(`[^-a-zA-Z0-9._]+`) + nonIdentifierChars = regexp.MustCompile(`[^a-zA-Z0-9_]+`) +) + +func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) { + indirectSortDirective, err := checkForIndirectSortDirective(lo) + if err != nil { + return nil, err + } + joinTableIndexByLabelName := make(map[string]int) + if indirectSortDirective != nil && isLabelsFieldList(indirectSortDirective.Fields) { + return l.constructIndirectSortQuery(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + } + ensureSortLabelsAreSelected(lo) + return l.finishConstructQuery(lo, partitions, namespace, dbName, joinTableIndexByLabelName) +} + +/** constructIndirectSortQuery - process indirect-sorting + * Here we create two queries: + * one that has an existence test for the sorter, + * and one with a non-existence test, so each of these is AND-ed with the other WHERE tests (filters). + * Then do a `UNION ALL` on the two different queries. + * The unobvious part: have the original options-list do only the one indirect sort. + * Have the copy process any other sort options. + * + * Two limitations: + * 1. Only at most one indirect sort per query + * 2. The indirect sort will go before the other ones (todo: fix this) + */ + +func (l *ListOptionIndexer) constructIndirectSortQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) (*QueryInfo, error) { + var loNoLabel sqltypes.ListOptions + // We want to make sure that the want-sort-label options test for the label's existence, + // but we want the non-label to have a not-exists test on it. So first ensure it exists, + // then ensure a non-exists test exists on the non-label filter. + // The other thing is we put all the non-indirect sort directives on the copy of the list options + var indirectSortDirective sqltypes.Sort + newSortList1 := make([]sqltypes.Sort, 1) + newSortList2 := make([]sqltypes.Sort, 0, len(lo.SortList.SortDirectives)-1) + indirectSortPosition := -1 + for i, sd := range lo.SortList.SortDirectives { + if sd.IsIndirect { + indirectSortDirective = lo.SortList.SortDirectives[i] + newSortList1[0] = indirectSortDirective + indirectSortPosition = i + } else { + newSortList2 = append(newSortList2, sd) + } + } + if indirectSortPosition == -1 { + return nil, fmt.Errorf("expected an indirect sort directive, didn't find one") + } + if len(indirectSortDirective.IndirectFields) != 4 { + return nil, fmt.Errorf("expected indirect sort directive to have 4 indirect fields, got %d", len(indirectSortDirective.IndirectFields)) + } + bytes, err := json.Marshal(*lo) + if err != nil { + return nil, fmt.Errorf("can't json-encode list options: %w", err) + } + err = json.Unmarshal(bytes, &loNoLabel) + if err != nil { + return nil, fmt.Errorf("can't json-decode list options: %w", err) + } + applyIndirectLabelTests(lo, &loNoLabel, &indirectSortDirective) + lo.SortList.SortDirectives = newSortList1 + loNoLabel.SortList.SortDirectives = newSortList2 + joinParts1, whereClauses1, params1, needsDistinctModifier1, _, _, _, err1 := l.getQueryParts(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + if err1 != nil { + return nil, err1 + } + // Now add clauses for the indirectSortDirective + joinParts2, whereClauses2, params2, needsDistinctModifier2, orderByClauses2, orderByParams2, _, err2 := l.getQueryParts(&loNoLabel, partitions, namespace, dbName, joinTableIndexByLabelName) + if err2 != nil { + return nil, err2 + } + addFalseTest := false + if whereClauses1[len(whereClauses1)-1] == "FALSE" { + whereClauses1 = whereClauses1[:len(whereClauses1)-1] + addFalseTest = true + } + if whereClauses2[len(whereClauses2)-1] == "FALSE" { + whereClauses2 = whereClauses2[:len(whereClauses2)-1] + addFalseTest = true + } + distinctModifier := "" + if needsDistinctModifier1 || needsDistinctModifier2 { + distinctModifier = " DISTINCT" + } + + externalTableName := getExternalTableName(&indirectSortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + return nil, fmt.Errorf("internal error: unable to find an entry for external table %s", externalTableName) + } + sortParts, importWithParts, importAsNullParts := processOrderByFields(&indirectSortDirective, extIndex, orderByClauses2) + selectLine := fmt.Sprintf("SELECT%s o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid", distinctModifier) + indent1 := " " + indent2 := indent1 + indent1 + indent3 := indent2 + indent1 + where1 := joinWhereClauses(whereClauses1, indent2, indent3, "AND") + where2 := joinWhereClauses(whereClauses2, indent2, indent3, "AND") + + parts := []string{ + "SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM (", + fmt.Sprintf(`%s%s, %s FROM %s`, indent1, selectLine, strings.Join(importWithParts, ", "), strings.Join(joinParts1, "\n"+indent2)), + where1, + "UNION ALL", + fmt.Sprintf(`%s%s, %s FROM %s`, indent1, selectLine, strings.Join(importAsNullParts, ", "), strings.Join(joinParts2, "\n"+indent2)), + where2, + ")", + } + if addFalseTest { + parts = append(parts, "WHERE FALSE") + } + params := make([]any, 0, len(params1)+len(params2)+len(orderByParams2)) + params = append(params, params1...) + params = append(params, params2...) + fullQuery := strings.Join(parts, "\n") + countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", fullQuery) + countParams := params[:] + params = append(params, orderByParams2...) + + fixedSortParts := putIndirectSortInPosition(sortParts, indirectSortPosition) + fullQuery += fmt.Sprintf("\n%sORDER BY %s", indent1, strings.Join(fixedSortParts, ", ")) + queryInfo := &QueryInfo{ + query: fullQuery, + params: params, + countQuery: countQuery, + countParams: countParams, + } + return queryInfo, nil +} + +func putIndirectSortInPosition(sortParts []string, indirectSortPosition int) []string { + fixedSortParts := make([]string, 0, len(sortParts)) + indirectSortPart := sortParts[0] + sortParts = sortParts[1:] + fixedSortParts = append(fixedSortParts, sortParts[0:indirectSortPosition]...) + fixedSortParts = append(fixedSortParts, indirectSortPart) + fixedSortParts = append(fixedSortParts, sortParts[indirectSortPosition:len(sortParts)]...) + return fixedSortParts +} + +func (l *ListOptionIndexer) finishConstructQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) (*QueryInfo, error) { + + joinParts, whereClauses, params, needsDistinctModifier, orderByClauses, orderByParams, sortSelectField, err := l.getQueryParts(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + if err != nil { + return nil, err + } + distinctModifier := "" + if needsDistinctModifier { + distinctModifier = " DISTINCT" + } + queryInfo := &QueryInfo{} + + if len(sortSelectField) > 0 { + if sortSelectField[0] != ' ' { + sortSelectField = " " + sortSelectField + } + } + query := fmt.Sprintf(`SELECT%s o.object, o.objectnonce, o.dekid%s FROM `, distinctModifier, sortSelectField) + query += strings.Join(joinParts, "\n ") + + if len(whereClauses) > 0 { + indent := " " + separator := fmt.Sprintf(") AND\n%s(", indent) + query += fmt.Sprintf("\n WHERE\n%s(%s)", indent, strings.Join(whereClauses, separator)) + } + + // before proceeding, save a copy of the query and params without LIMIT/OFFSET/ORDER info + // for COUNTing all results later + countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", query) + countParams := params[:] + if len(orderByClauses) > 0 { + query += "\n ORDER BY " + query += strings.Join(orderByClauses, ", ") + params = append(params, orderByParams...) + } + + // 4- sqltypes.Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume) + + limitClause := "" + // take the smallest limit between lo.Pagination and lo.ChunkSize + limit := lo.Pagination.PageSize + if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) { + limit = lo.ChunkSize + } + if limit > 0 { + limitClause = "\n LIMIT ?" + params = append(params, limit) + } + + // OFFSET clause (from lo.Pagination and/or lo.Resume) + offsetClause := "" + offset := 0 + if lo.Resume != "" { + offsetInt, err := strconv.Atoi(lo.Resume) + if err != nil { + return queryInfo, err + } + offset = offsetInt + } + if lo.Pagination.Page >= 1 { + offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1) + } + if offset > 0 { + offsetClause = "\n OFFSET ?" + params = append(params, offset) + } + if limit > 0 || offset > 0 { + query += limitClause + query += offsetClause + queryInfo.countQuery = countQuery + queryInfo.countParams = countParams + queryInfo.limit = limit + queryInfo.offset = offset + } + // Otherwise leave these as default values and the executor won't do pagination work + + queryInfo.query = query + queryInfo.params = params + + return queryInfo, nil +} + +// Other ListOptionIndexer methods for generating SQL in alphabetical order: + +// buildORClause creates an SQLite compatible query that ORs conditions built from passed filters +func (l *ListOptionIndexer) buildORClauseFromFilters(orFilters sqltypes.OrFilter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, bool, error) { + params := make([]any, 0) + whereClauses := make([]string, 0, len(orFilters.Filters)) + joinClauses := make([]string, 0) + needDistinct := false + + for _, filter := range orFilters.Filters { + if isLabelFilter(&filter) { + fullName := fmt.Sprintf("%s:%s", dbName, filter.Field[2]) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + labelIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[fullName] = labelIndex + } + _, ok = joinedTables[fullName] + if !ok { + joinedTables[fullName] = true + joinClauses = append(joinClauses, fmt.Sprintf(`LEFT OUTER JOIN "%s_labels" lt%d ON o.key = lt%d.key`, dbName, labelIndex, labelIndex)) + } + needDistinct = true + labelFunc := l.getLabelFilter + if isIndirectFilter(&filter) { + labelFunc = l.getIndirectLabelFilter + } + newWhereClause, newJoins, newParams, err := labelFunc(filter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, needDistinct, err + } + joinClauses = append(joinClauses, newJoins...) + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } else if isIndirectFilter(&filter) { + newWhereClause, newJoins, newParams, err := l.getIndirectNonLabelFilter(filter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, needDistinct, err + } + joinClauses = append(joinClauses, newJoins...) + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } else { + newWhereClause, newParams, err := l.getFieldFilter(filter) + if err != nil { + return "", nil, nil, needDistinct, err + } + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } + } + finalWhereClause := "" + switch len(whereClauses) { + case 0: + finalWhereClause = "" // no change + case 1: + finalWhereClause = whereClauses[0] + default: + finalWhereClause = fmt.Sprintf("(%s)", strings.Join(whereClauses, ") OR (")) + } + return finalWhereClause, joinClauses, params, needDistinct, nil +} + +// ensureSortLabelsAreSelected - if the user tries to sort on a particular label without mentioning it in a query, +// and it's not an indirect sort directive, we need to ensure the label is added. +func ensureSortLabelsAreSelected(lo *sqltypes.ListOptions) { + if len(lo.SortList.SortDirectives) == 0 { + return + } + unboundSortLabels := make(map[string]bool) + for _, sortDirective := range lo.SortList.SortDirectives { + fields := sortDirective.Fields + if isLabelsFieldList(fields) { + unboundSortLabels[fields[2]] = true + } + } + if len(unboundSortLabels) == 0 { + return + } + // If we have sort directives but no filters, add an exists-filter for each label. + if lo.Filters == nil || len(lo.Filters) == 0 { + lo.Filters = make([]sqltypes.OrFilter, 1) + lo.Filters[0].Filters = make([]sqltypes.Filter, len(unboundSortLabels)) + i := 0 + for labelName := range unboundSortLabels { + lo.Filters[0].Filters[i] = sqltypes.Filter{ + Field: []string{"metadata", "labels", labelName}, + Op: sqltypes.Exists, + } + i++ + } + return + } + // Find any labels that are already mentioned in an existing filter + // The gotcha is we have to bind the labels for each set of orFilters, so copy them each time + for i, orFilters := range lo.Filters { + copyUnboundSortLabels := make(map[string]bool, len(unboundSortLabels)) + for k, v := range unboundSortLabels { + copyUnboundSortLabels[k] = v + } + for _, filter := range orFilters.Filters { + if isLabelFilter(&filter) { + copyUnboundSortLabels[filter.Field[2]] = false + } + } + // Now for any labels that are still true, add another where clause + for labelName, needsBinding := range copyUnboundSortLabels { + if needsBinding { + // `orFilters` is a copy of lo.Filters[i], so reference the original. + lo.Filters[i].Filters = append(lo.Filters[i].Filters, sqltypes.Filter{ + Field: []string{"metadata", "labels", labelName}, + Op: sqltypes.Exists, + }) + } + } + } +} + +// Possible ops from the k8s parser: +// KEY = and == (same) VALUE +// KEY != VALUE +// KEY exists [] # ,KEY, => this filter +// KEY ! [] # ,!KEY, => assert KEY doesn't exist +// KEY in VALUES +// KEY notin VALUES + +func (l *ListOptionIndexer) getFieldFilter(filter sqltypes.Filter) (string, []any, error) { + opString := "" + escapeString := "" + columnName := toColumnName(filter.Field) + if err := l.validateColumn(columnName); err != nil { + return "", nil, err + } + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + } else { + opString = "=" + } + clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) + return clause, []any{formatMatchTarget(filter)}, nil + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + } else { + opString = "!=" + } + clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) + return clause, []any{formatMatchTarget(filter)}, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, err + } + clause := fmt.Sprintf(`f."%s" %s ?`, columnName, sym) + return clause, []any{target}, nil + + case sqltypes.Exists, sqltypes.NotExists: + return "", nil, errors.New("NULL and NOT NULL tests aren't supported for non-label queries") + + case sqltypes.In: + fallthrough + case sqltypes.NotIn: + target := "()" + if len(filter.Matches) > 0 { + target = fmt.Sprintf("(?%s)", strings.Repeat(", ?", len(filter.Matches)-1)) + } + opString = "IN" + if filter.Op == sqltypes.NotIn { + opString = "NOT IN" + } + clause := fmt.Sprintf(`f."%s" %s %s`, columnName, opString, target) + matches := make([]any, len(filter.Matches)) + for i, match := range filter.Matches { + matches[i] = match + } + return clause, matches, nil + } + + return "", nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getIndirectLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + if len(filter.IndirectFields) != 4 { + s := "" + if len(filter.IndirectFields) > 0 { + s = strings.Join(filter.IndirectFields, " ") + } + return "", nil, nil, fmt.Errorf("expected exactly 4 indirect field parts, got %s (%d)", s, len(filter.IndirectFields)) + } + labelName := filter.Field[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", nil, nil, fmt.Errorf("internal error: can't find an entry for table %s", fullName) + } + + joinClauses := make([]string, 0) + + extDBName := fmt.Sprintf("%s_%s", filter.IndirectFields[0], filter.IndirectFields[1]) + extDBName = badTableNameChars.ReplaceAllString(extDBName, "_") + extDBName = strings.ReplaceAll(extDBName, "/", "_") + extIndex, ok := joinTableIndexByLabelName[extDBName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[extDBName] = extIndex + } + + selectorFieldName := filter.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + targetFieldName := filter.IndirectFields[3] + if badTableNameChars.MatchString(targetFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", targetFieldName) + } + extDBNameFields := fmt.Sprintf("%s_fields", extDBName) + _, ok = joinedTables[extDBNameFields] + if !ok { + joinedTables[extDBNameFields] = true + joinClauses = append(joinClauses, fmt.Sprintf(`JOIN "%s" ext%d ON lt%d.value = ext%d."%s"`, extDBNameFields, extIndex, labelIndex, extIndex, selectorFieldName)) + } + labelWhereSubClause := fmt.Sprintf("lt%d.label = ?", labelIndex) + targetFieldReference := fmt.Sprintf(`ext%d."%s"`, extIndex, targetFieldName) + var clause string + var op string + params := []any{labelName} + + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause = fmt.Sprintf(`%s AND %s %s ?%s`, labelWhereSubClause, targetFieldReference, opString, escapeString) + params = append(params, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + clause = fmt.Sprintf(`%s AND %s %s ?%s`, labelWhereSubClause, targetFieldReference, opString, escapeString) + params = append(params, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`%s AND %s %s ?`, labelWhereSubClause, targetFieldReference, sym) + params = append(params, target) + return clause, joinClauses, params, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`%s AND %s != NULL`, labelWhereSubClause, targetFieldReference) + return clause, joinClauses, params, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`%s AND %s == NULL`, labelWhereSubClause, targetFieldReference) + return clause, joinClauses, params, nil + + case sqltypes.In, sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + op = "IN" + if filter.Op == sqltypes.NotIn { + op = "NOT IN" + } + clause := fmt.Sprintf(`%s AND %s %s %s`, labelWhereSubClause, targetFieldReference, op, target) + for _, match := range filter.Matches { + params = append(params, match) + } + return clause, joinClauses, params, nil + + // See getLabelFilter for rest of operators + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getIndirectNonLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + if len(filter.IndirectFields) != 4 { + s := "" + if len(filter.IndirectFields) > 0 { + s = strings.Join(filter.IndirectFields, " ") + } + return "", nil, nil, fmt.Errorf("expected exactly 4 indirect field parts, got %s (%d)", s, len(filter.IndirectFields)) + } + columnName := toColumnName(filter.Field) + if err := l.validateColumn(columnName); err != nil { + return "", nil, nil, err + } + extDBName := fmt.Sprintf("%s_%s", filter.IndirectFields[0], filter.IndirectFields[1]) + extDBName = badTableNameChars.ReplaceAllString(extDBName, "_") + extIndex, ok := joinTableIndexByLabelName[extDBName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[extDBName] = extIndex + } + + selectorFieldName := filter.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := filter.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + extDBNameFields := fmt.Sprintf("%s_fields", extDBName) + _, ok = joinedTables[extDBNameFields] + joinClauses := make([]string, 0) + if !ok { + joinedTables[extDBNameFields] = true + joinClauses = append(joinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON f."%s" = ext%d."%s"`, extDBName, extIndex, columnName, extIndex, selectorFieldName)) + } + params := make([]any, 0) + + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause := fmt.Sprintf(`ext%d."%s" %s ?%s`, extIndex, externalFieldName, opString, escapeString) + return clause, joinClauses, []any{formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + clause := fmt.Sprintf(`ext%d."%s" %s ?%s`, extIndex, externalFieldName, opString, escapeString) + return clause, joinClauses, []any{formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`ext%d."%s" %s ?`, extIndex, externalFieldName, sym) + return clause, joinClauses, []any{target}, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`ext%d."%s" != NULL`, extIndex, externalFieldName) + return clause, joinClauses, []any{}, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`ext%d."%s" == NULL`, extIndex, externalFieldName) + return clause, joinClauses, []any{}, nil + + case sqltypes.In, sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + opString = "IN" + if filter.Op == sqltypes.NotIn { + opString = "NOT IN" + } + clause := fmt.Sprintf(`ext%d."%s" %s %s`, extIndex, externalFieldName, opString, target) + for _, match := range filter.Matches { + params = append(params, match) + } + return clause, joinClauses, params, nil + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + labelName := filter.Field[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", nil, nil, fmt.Errorf("internal error: can't find an entry for table %s", fullName) + } + + joinClauses := make([]string, 0) + + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?%s`, labelIndex, labelIndex, opString, escapeString) + return clause, joinClauses, []any{labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + subFilter := sqltypes.Filter{ + Field: filter.Field, + Op: sqltypes.NotExists, + } + existenceClause, _, subParams, err := l.getLabelFilter(subFilter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value %s ?%s)`, existenceClause, labelIndex, labelIndex, opString, escapeString) + params := append(subParams, labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?`, labelIndex, labelIndex, sym) + return clause, joinClauses, []any{labelName, target}, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`lt%d.label = ?`, labelIndex) + return clause, joinClauses, []any{labelName}, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`o.key NOT IN (SELECT o1.key FROM "%s" o1 + JOIN "%s_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "%s_labels" lt%di1 ON o1.key = lt%di1.key + WHERE lt%di1.label = ?)`, dbName, dbName, dbName, labelIndex, labelIndex, labelIndex) + return clause, joinClauses, []any{labelName}, nil + + case sqltypes.In: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value IN %s`, labelIndex, labelIndex, target) + matches := make([]any, len(filter.Matches)+1) + matches[0] = labelName + for i, match := range filter.Matches { + matches[i+1] = match + } + return clause, joinClauses, matches, nil + + case sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + subFilter := sqltypes.Filter{ + Field: filter.Field, + Op: sqltypes.NotExists, + } + existenceClause, _, subParams, err := l.getLabelFilter(subFilter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value NOT IN %s)`, existenceClause, labelIndex, labelIndex, target) + matches := append(subParams, labelName) + for _, match := range filter.Matches { + matches = append(matches, match) + } + return clause, joinClauses, matches, nil + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getQueryParts(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) ([]string, []string, []any, bool, []string, []any, string, error) { + joinParts := []string{fmt.Sprintf(`"%s" o`, dbName), fmt.Sprintf(`JOIN "%s_fields" f ON o.key = f.key`, dbName)} + whereClauses := make([]string, 0) + params := make([]any, 0) + needDistinctFinal := false + joinedTables := make(map[string]bool) + joinedTables[dbName] = true + joinedTables[fmt.Sprintf("%s_fields", dbName)] = true + // 1- Figure out what we'll be joining and testing + for _, orFilters := range lo.Filters { + newWhereClause, newJoinParts, newParams, needDistinct, err := l.buildORClauseFromFilters(orFilters, dbName, joinTableIndexByLabelName, joinedTables) + + if err != nil { + return joinParts, whereClauses, params, needDistinctFinal, nil, nil, "", err + } + joinParts = append(joinParts, newJoinParts...) + if len(newWhereClause) > 0 { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + if needDistinct { + needDistinctFinal = true + } + } + + // WHERE clauses (from namespace) + if namespace != "" && namespace != "*" { + whereClauses = append(whereClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) + params = append(params, namespace) + } + + // WHERE clauses (from partitions and their corresponding parameters) + partitionClauses := make([]string, 0) + for _, thisPartition := range partitions { + if thisPartition.Passthrough { + // nothing to do, no extra filtering to apply by definition + } else { + singlePartitionClauses := make([]string, 0) + + // filter by namespace + if thisPartition.Namespace != "" && thisPartition.Namespace != "*" { + singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) + params = append(params, thisPartition.Namespace) + } + + // optionally filter by names + if !thisPartition.All { + names := thisPartition.Names + + if names.Len() == 0 { + // degenerate case, there will be no results + singlePartitionClauses = append(singlePartitionClauses, "FALSE") + } else { + singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.name" IN (?%s)`, strings.Repeat(", ?", thisPartition.Names.Len()-1))) + // sort for reproducibility + sortedNames := thisPartition.Names.UnsortedList() + sort.Strings(sortedNames) + for _, name := range sortedNames { + params = append(params, name) + } + } + } + + if len(singlePartitionClauses) > 0 { + partitionClauses = append(partitionClauses, strings.Join(singlePartitionClauses, " AND ")) + } + } + } + if len(partitions) == 0 { + // degenerate case, there will be no results + whereClauses = append(whereClauses, "FALSE") + } + if len(partitionClauses) == 1 { + whereClauses = append(whereClauses, partitionClauses[0]) + } + if len(partitionClauses) > 1 { + whereClauses = append(whereClauses, "(\n ("+strings.Join(partitionClauses, ") OR\n (")+")\n)") + } + sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err := l.getSortDirectives(lo, dbName, joinTableIndexByLabelName) + joinParts = append(joinParts, sortJoinClauses...) + whereClauses = append(whereClauses, sortWhereClauses...) + + return joinParts, whereClauses, params, needDistinctFinal, orderByClauses, orderByParams, sortSelectField, err +} + +func (l *ListOptionIndexer) getSortDirectives(lo *sqltypes.ListOptions, dbName string, joinTableIndexByLabelName map[string]int) (string, []string, []string, []string, []any, error) { + sortSelectField := "" + sortJoinClauses := make([]string, 0) + sortWhereClauses := make([]string, 0) + orderByClauses := make([]string, 0) + orderByParams := make([]any, 0) + if len(lo.SortList.SortDirectives) == 0 { + // make sure at least one default order is always picked + orderByClauses = append(orderByClauses, `f."metadata.name" ASC`) + if l.namespaced { + orderByClauses = append(orderByClauses, `f."metadata.namespace" ASC`) + } + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, nil + } + for _, sortDirective := range lo.SortList.SortDirectives { + fields := sortDirective.Fields + if isLabelsFieldList(fields) { + labelName := sortDirective.Fields[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + if sortDirective.IsIndirect { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) + } + labelIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[fullName] = labelIndex + } + if sortDirective.IsIndirect { + //TODO: check the external table name. + externalTableName := getExternalTableName(&sortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[externalTableName] = extIndex + } + selectorFieldName := sortDirective.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := sortDirective.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + sortJoinClauses = append(sortJoinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON lt%d.value = ext%d."%s"`, externalTableName, extIndex, labelIndex, extIndex, selectorFieldName)) + //TODO: Verify the field name + sortSelectField = fmt.Sprintf(`ext%d."%s" as ext%d_target`, extIndex, externalFieldName, extIndex) + + } + clause, sortParam, err := buildSortLabelsClause(fields[2], dbName, joinTableIndexByLabelName, sortDirective.Order == sqltypes.ASC) + if err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + orderByClauses = append(orderByClauses, clause) + orderByParams = append(orderByParams, sortParam) + } else if sortDirective.IsIndirect { + if len(sortDirective.IndirectFields) != 4 { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("expected indirect sort directive to have 4 indirect fields, got %d", len(sortDirective.IndirectFields)) + } + externalTableName := getExternalTableName(&sortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[externalTableName] = extIndex + } + selectorFieldName := sortDirective.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := sortDirective.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + columnName := toColumnName(fields) + if err := l.validateColumn(columnName); err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + sortJoinClauses = append(sortJoinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON f."%s" = ext%d."%s"`, externalTableName, extIndex, columnName, extIndex, selectorFieldName)) + direction := "ASC" + nullsPlace := "LAST" + if sortDirective.Order == sqltypes.DESC { + direction = "DESC" + nullsPlace = "FIRST" + } + orderByClauses = append(orderByClauses, fmt.Sprintf(`ext%d."%s" %s NULLS %s`, extIndex, externalFieldName, direction, nullsPlace)) + } else { + columnName := toColumnName(fields) + if err := l.validateColumn(columnName); err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + direction := "ASC" + if sortDirective.Order == sqltypes.DESC { + direction = "DESC" + } + orderByClauses = append(orderByClauses, fmt.Sprintf(`f."%s" %s`, columnName, direction)) + } + } + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, nil +} + +func (l *ListOptionIndexer) validateColumn(column string) error { + for _, v := range l.indexedFields { + if v == column { + return nil + } + } + return fmt.Errorf("column is invalid [%s]: %w", column, ErrInvalidColumn) +} + +// Helper functions for generating SQL in alphabetical order: + +func applyIndirectLabelTests(loWithLabel *sqltypes.ListOptions, loWithoutLabel *sqltypes.ListOptions, indirectSortDirective *sqltypes.Sort) { + labelFilter := sqltypes.Filter{ + Field: indirectSortDirective.Fields[:], + //Matches: make([]string, 0), + Op: sqltypes.Exists, + } + loWithLabel.Filters = append(loWithLabel.Filters, sqltypes.OrFilter{Filters: []sqltypes.Filter{labelFilter}}) + + // And add an AND-test that the label does not exists for the second test + nonLabelFilter := labelFilter + nonLabelFilter.Op = sqltypes.NotExists + loWithoutLabel.Filters = append(loWithoutLabel.Filters, sqltypes.OrFilter{Filters: []sqltypes.Filter{nonLabelFilter}}) +} + +func buildSortLabelsClause(labelName string, dbName string, joinTableIndexByLabelName map[string]int, isAsc bool) (string, string, error) { + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", "", fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) + } + stmt := fmt.Sprintf(`CASE lt%d.label WHEN ? THEN lt%d.value ELSE NULL END`, labelIndex, labelIndex) + dir := "ASC" + nullsPosition := "LAST" + if !isAsc { + dir = "DESC" + nullsPosition = "FIRST" + } + return fmt.Sprintf("(%s) %s NULLS %s", stmt, dir, nullsPosition), labelName, nil +} + +func checkForIndirectSortDirective(lo *sqltypes.ListOptions) (*sqltypes.Sort, error) { + indirectSortDirectives := make([]string, 0) + var id *sqltypes.Sort + for _, sd := range lo.SortList.SortDirectives { + if sd.IsIndirect { + id = &sd + indirectSortDirectives = append(indirectSortDirectives, fmt.Sprintf("[%s]", strings.Join(sd.IndirectFields, "]["))) + } + } + if len(indirectSortDirectives) > 1 { + return nil, fmt.Errorf("can have at most one indirect sort directive, have %d: %s", len(indirectSortDirectives), indirectSortDirectives) + } + return id, nil +} + +func formatMatchTarget(filter sqltypes.Filter) string { + format := strictMatchFmt + if filter.Partial { + format = matchFmt + } + return formatMatchTargetWithFormatter(filter.Matches[0], format) +} + +func formatMatchTargetWithFormatter(match string, format string) string { + // To allow matches on the backslash itself, the character needs to be replaced first. + // Otherwise, it will undo the following replacements. + match = strings.ReplaceAll(match, `\`, `\\`) + match = strings.ReplaceAll(match, `_`, `\_`) + match = strings.ReplaceAll(match, `%`, `\%`) + return fmt.Sprintf(format, match) +} + +func getExternalTableName(sd *sqltypes.Sort) string { + s := strings.Join(sd.IndirectFields[0:2], "_") + return strings.ReplaceAll(s, "/", "_") +} + +func isIndirectFilter(filter *sqltypes.Filter) bool { + return filter.IsIndirect +} + +func isLabelFilter(f *sqltypes.Filter) bool { + return len(f.Field) >= 2 && f.Field[0] == "metadata" && f.Field[1] == "labels" +} + +func isLabelsFieldList(fields []string) bool { + return len(fields) == 3 && fields[0] == "metadata" && fields[1] == "labels" +} + +func joinWhereClauses(whereClauses []string, leadingIndent string, continuingIndent string, op string) string { + switch len(whereClauses) { + case 0: + return "" + case 1: + return fmt.Sprintf("%sWHERE %s\n", leadingIndent, whereClauses[0]) + } + separator := fmt.Sprintf(") %s\n%s(", op, continuingIndent) + return fmt.Sprintf("%sWHERE (%s)\n", leadingIndent, strings.Join(whereClauses, separator)) +} + +func prepareComparisonParameters(op sqltypes.Op, target string) (string, float64, error) { + num, err := strconv.ParseFloat(target, 32) + if err != nil { + return "", 0, err + } + switch op { + case sqltypes.Lt: + return "<", num, nil + case sqltypes.Gt: + return ">", num, nil + } + return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op) +} + +func processOrderByFields(sd *sqltypes.Sort, extIndex int, orderByClauses []string) ([]string, []string, []string) { + sortFieldMap := make(map[string]string) + externalFieldName := sd.IndirectFields[3] + newName := fmt.Sprintf("__ix_ext%d_%s", extIndex, nonIdentifierChars.ReplaceAllString(externalFieldName, "_")) + sortFieldMap[externalFieldName] = newName + sortParts := make([]string, 1+len(orderByClauses)) + direction := "ASC" + nullPosition := "LAST" + if sd.Order == sqltypes.DESC { + direction = "DESC" + nullPosition = "FIRST" + } + sortParts[0] = fmt.Sprintf("%s %s NULLS %s", newName, direction, nullPosition) + importWithParts := make([]string, 1+len(orderByClauses)) + importWithParts[0] = fmt.Sprintf(`ext%d."%s" AS %s`, extIndex, externalFieldName, newName) + importAsNullParts := make([]string, 1+len(orderByClauses)) + importAsNullParts[0] = fmt.Sprintf("NULL AS %s", newName) + for i, clause := range orderByClauses { + orderParts := strings.SplitN(clause, " ", 2) + fieldName := orderParts[0] + _, ok := sortFieldMap[fieldName] + if ok { + continue + } + fieldParts := strings.SplitN(fieldName, ".", 2) + prefix := fieldParts[0] + baseName := fieldParts[1] + if baseName[0] == '"' { + baseName = baseName[1 : len(baseName)-1] + } + newBaseName := nonIdentifierChars.ReplaceAllString(baseName, "_") + newName := fmt.Sprintf("__ix_%s_%s", prefix, newBaseName) + sortFieldMap[fieldName] = newName + importWithParts[i+1] = fmt.Sprintf("%s AS %s", fieldName, newName) + importAsNullParts[i+1] = importWithParts[i+1] + sortParts[i+1] = fmt.Sprintf("%s %s", newName, orderParts[1]) + } + return sortParts, importWithParts, importAsNullParts +} + +// There are two kinds of string arrays to turn into a string, based on the last value in the array +// simple: ["a", "b", "conformsToIdentifier"] => "a.b.conformsToIdentifier" +// complex: ["a", "b", "foo.io/stuff"] => "a.b[foo.io/stuff]" + +func smartJoin(s []string) string { + if len(s) == 0 { + return "" + } + if len(s) == 1 { + return s[0] + } + lastBit := s[len(s)-1] + simpleName := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + if simpleName.MatchString(lastBit) { + return strings.Join(s, ".") + } + return fmt.Sprintf("%s[%s]", strings.Join(s[0:len(s)-1], "."), lastBit) +} diff --git a/pkg/sqlcache/informer/query_generator_test.go b/pkg/sqlcache/informer/query_generator_test.go new file mode 100644 index 00000000..ff191edf --- /dev/null +++ b/pkg/sqlcache/informer/query_generator_test.go @@ -0,0 +1,2752 @@ +/* +Copyright 2024, 2025 SUSE LLC +*/ + +package informer + +import ( + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/rancher/steve/pkg/sqlcache/db" + "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestListByOptions(t *testing.T) { + type testCase struct { + description string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + extraIndexedFields []string + expectedList *unstructured.UnstructuredList + returnList []any + expectedContToken string + expectedErr error + } + + testObject := testStoreObject{Id: "something", Val: "a"} + unstrTestObjectMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testObject) + assert.Nil(t, err) + + var tests []testCase + tests = append(tests, testCase{ + description: "ListByOptions() with no errors returned, should not return an error", + listOptions: sqltypes.ListOptions{}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions() with an empty filter, should not return an error", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{{[]sqltypes.Filter{}}}, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ChunkSize: 2}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ?`, + expectedStmtArgs: []interface{}{2}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with Resume set should set offset in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Resume: "4"}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + OFFSET ?`, + expectedStmtArgs: []interface{}{4}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 OrFilter set with 1 filter with Op set top NotEq should select where that filter is not true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" NOT LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 sqltypes.OrFilter set with 1 filter with Partial set to true should select where that partial match on that filter's value is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 sqltypes.OrFilter set with multiple filters should select where any of those filters are true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"someothervalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somethirdvalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + ((f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" NOT LIKE ? ESCAPE '\')) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%", "%someothervalue%", "%somethirdvalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with multiple OrFilters set should select where all OrFilters contain one filter that is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"value1"}, + Op: sqltypes.Eq, + Partial: false, + }, + { + Field: []string{"status", "someotherfield"}, + Matches: []string{"value2"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"value3"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test4", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + ((f."metadata.somefield" = ?) OR (f."status.someotherfield" != ?)) AND + (f."metadata.somefield" = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"value1", "value2", "value3", "test4"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with labels filter should select the label in the prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "guard.cattle.io"}, + Matches: []string{"lodgepole"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test41", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"guard.cattle.io", "%lodgepole%", "test41"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with two labels filters should use a self-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"milk"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "horses"}, + Matches: []string{"saddles"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test42", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (lt2.label = ? AND lt2.value = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"cows", "milk", "horses", "saddles", "test42"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with a mix of one label and one non-label query can still self-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"butter"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"wheat"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test43", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (f."metadata.somefield" = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"cows", "butter", "wheat", "test43"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with only one sqltypes.Sort.Field set should sort on that field only, in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.somefield" ASC`, + expectedStmtArgs: []any{"test5"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "sort one field descending", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5a", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.somefield" DESC`, + expectedStmtArgs: []any{"test5a"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "sort one unbound label descending", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "flip"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5a", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) DESC NULLS FIRST`, + expectedStmtArgs: []any{"flip", "test5a", "flip"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions sorting on two complex fields should sort on the first field in ascending order first and then sort on the second labels field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "fields", "3"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"metadata", "labels", "stub.io/candy"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + extraIndexedFields: []string{"metadata.fields[3]", "metadata.labels[stub.io/candy]"}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY f."metadata.fields[3]" ASC, (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, + expectedStmtArgs: []any{"stub.io/candy", "stub.io/candy"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions sorting on two fields should sort on the first field in ascending order first and then sort on the second field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"status", "someotherfield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.somefield" ASC, f."status.someotherfield" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions sorting on two fields should sort on the first field in descending order first and then sort on the second field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.DESC, + }, + { + Fields: []string{"status", "someotherfield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.somefield" DESC, f."status.someotherfield" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.PageSize set should set limit to PageSize in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + PageSize: 10, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ?`, + expectedStmtArgs: []any{10}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.Page and no PageSize set should not add anything to prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + Page: 2, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.Page and PageSize set limit to PageSize and offset to PageSize * (Page - 1) in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + PageSize: 10, + Page: 2, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ? + OFFSET ?`, + expectedStmtArgs: []any{10, 10}, + + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Namespace Partition should select only items where metadata.namespace is equal to Namespace and all other conditions are met in prepared sql.Stmt", + partitions: []partition.Partition{ + { + Namespace: "somens", + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ? AND FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somens"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a All Partition should select all items that meet all other conditions in prepared sql.Stmt", + partitions: []partition.Partition{ + { + All: true, + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Passthrough Partition should select all items that meet all other conditions prepared sql.Stmt", + partitions: []partition.Partition{ + { + Passthrough: true, + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Names Partition should select only items where metadata.name equals an items in Names and all other conditions are met in prepared sql.Stmt", + partitions: []partition.Partition{ + { + Names: sets.New[string]("someid", "someotherid"), + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.name" IN (?, ?)) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"someid", "someotherid"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + txClient := NewMockTXClient(gomock.NewController(t)) + store := NewMockStore(gomock.NewController(t)) + stmts := NewMockStmt(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.somefield", "status.someotherfield"}, + } + if len(test.extraIndexedFields) > 0 { + lii.indexedFields = append(lii.indexedFields, test.extraIndexedFields...) + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") + if test.expectedErr != nil { + assert.Equal(t, test.expectedErr, err) + return + } + assert.Nil(t, err) + assert.Equal(t, test.expectedStmt, queryInfo.query) + if test.expectedStmtArgs == nil { + test.expectedStmtArgs = []any{} + } + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + + stmt := &sql.Stmt{} + rows := &sql.Rows{} + objType := reflect.TypeOf(testObject) + txClient.EXPECT().Stmt(gomock.Any()).Return(stmts).AnyTimes() + store.EXPECT().Prepare(test.expectedStmt).Do(func(a ...any) { + fmt.Println(a) + }).Return(stmt) + if args := test.expectedStmtArgs; args != nil { + stmts.EXPECT().QueryContext(gomock.Any(), gomock.Any()).Return(rows, nil).AnyTimes() + } else if strings.Contains(test.expectedStmt, "LIMIT") { + stmts.EXPECT().QueryContext(gomock.Any(), args...).Return(rows, nil) + txClient.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) + } else { + stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) + } + store.EXPECT().GetType().Return(objType) + store.EXPECT().GetShouldEncrypt().Return(false) + store.EXPECT().ReadObjects(rows, objType, false).Return(test.returnList, nil) + store.EXPECT().CloseStmt(stmt).Return(nil) + + store.EXPECT().WithTransaction(gomock.Any(), false, gomock.Any()).Return(nil).Do( + func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { + err := f(txClient) + if test.expectedErr == nil { + assert.Nil(t, err) + } else { + assert.Equal(t, test.expectedErr, err) + } + }) + + if test.expectedCountStmt != "" { + store.EXPECT().Prepare(test.expectedCountStmt).Return(stmt) + store.EXPECT().ReadInt(rows).Return(len(test.expectedList.Items), nil) + store.EXPECT().CloseStmt(stmt).Return(nil) + } + list, total, contToken, err := lii.executeQuery(context.Background(), queryInfo) + if test.expectedErr == nil { + assert.Nil(t, err) + } else { + assert.Equal(t, test.expectedErr, err) + } + assert.Equal(t, test.expectedList, list) + assert.Equal(t, len(test.expectedList.Items), total) + assert.Equal(t, test.expectedContToken, contToken) + }) + } +} + +func TestConstructQuery(t *testing.T) { + type testCase struct { + description string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr error + } + + var tests []testCase + tests = append(tests, testCase{ + description: "TestConstructQuery: handles IN statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"somevalue"}, + Op: sqltypes.In, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.queryField1" IN (?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOT-IN statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotIn, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.queryField1" NOT IN (?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles EXISTS statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Exists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOT-EXISTS statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.NotExists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualFull"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelEqualFull", "somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements, match partial", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles != statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNotEqualFull"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNotEqualFull", "labelNotEqualFull", "somevalue"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles != statements for label statements, match partial", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNotEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT LIKE ? ESCAPE '\')) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"labelNotEqualPartial", "labelNotEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles multiple != statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "notEqual1"}, + Matches: []string{"value1"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "notEqual2"}, + Matches: []string{"value2"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt2i1 ON o1.key = lt2i1.key + WHERE lt2i1.label = ?)) OR (lt2.label = ? AND lt2.value != ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"notEqual1", "notEqual1", "value1", "notEqual2", "notEqual2", "value2"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles IN statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelIN"}, + Matches: []string{"somevalue1", "someValue2"}, + Op: sqltypes.In, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value IN (?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelIN", "somevalue1", "someValue2"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOTIN statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNOTIN"}, + Matches: []string{"somevalue1", "someValue2"}, + Op: sqltypes.NotIn, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT IN (?, ?))) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNOTIN", "labelNOTIN", "somevalue1", "someValue2"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles EXISTS statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEXISTS"}, + Matches: []string{}, + Op: sqltypes.Exists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelEXISTS"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOTEXISTS statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNOTEXISTS"}, + Matches: []string{}, + Op: sqltypes.NotExists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNOTEXISTS"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles LessThan statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "numericThing"}, + Matches: []string{"5"}, + Op: sqltypes.Lt, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"numericThing", float64(5)}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles GreaterThan statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "numericThing"}, + Matches: []string{"35"}, + Op: sqltypes.Gt, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"numericThing", float64(35)}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "multiple filters with a positive label test and a negative non-label test still outer-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "junta"}, + Matches: []string{"esther"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"golgi"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"status", "queryField2"}, + Matches: []string{"gold"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" NOT LIKE ? ESCAPE '\')) AND + (f."status.queryField2" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"junta", "%esther%", "%golgi%", "gold"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "multiple filters and or-filters with a positive label test and a negative non-label test still outer-join and have correct AND/ORs", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "nectar"}, + Matches: []string{"stash"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"landlady"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "lawn"}, + Matches: []string{"reba", "coil"}, + Op: sqltypes.In, + }, + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Gt, + Matches: []string{"2"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" != ?)) AND + ((lt2.label = ? AND lt2.value IN (?, ?)) OR (f."metadata.queryField1" > ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"nectar", "%stash%", "landlady", "lawn", "reba", "coil", float64(2)}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements, match partial, sort on metadata.queryField1", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.queryField1" ASC`, + expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: sort on label statements with no query", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "this"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, //' + expectedStmtArgs: []any{"this", "this"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: sort and query on both labels and non-labels without overlap", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"toys"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "labels", "jamb"}, + Matches: []string{"juice"}, + Op: sqltypes.Eq, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "this"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"status", "queryField2"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((f."metadata.queryField1" = ?) OR (lt1.label = ? AND lt1.value = ?) OR (lt2.label = ?)) AND + (FALSE) + ORDER BY (CASE lt2.label WHEN ? THEN lt2.value ELSE NULL END) ASC NULLS LAST, f."status.queryField2" DESC`, + expectedStmtArgs: []any{"toys", "jamb", "juice", "this", "this"}, + expectedErr: nil, + }) + + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") + if test.expectedErr != nil { + assert.Equal(t, test.expectedErr, err) + return + } + require.Nil(t, err) + assert.Equal(t, test.expectedStmt, queryInfo.query) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestSmartJoin(t *testing.T) { + type testCase struct { + description string + fieldArray []string + expectedFieldName string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "single-letter names should be dotted", + fieldArray: []string{"metadata", "labels", "a"}, + expectedFieldName: "metadata.labels.a", + }) + tests = append(tests, testCase{ + description: "underscore should be dotted", + fieldArray: []string{"metadata", "labels", "_"}, + expectedFieldName: "metadata.labels._", + }) + tests = append(tests, testCase{ + description: "simple names should be dotted", + fieldArray: []string{"metadata", "labels", "queryField2"}, + expectedFieldName: "metadata.labels.queryField2", + }) + tests = append(tests, testCase{ + description: "a numeric field should be bracketed", + fieldArray: []string{"metadata", "fields", "43"}, + expectedFieldName: "metadata.fields[43]", + }) + tests = append(tests, testCase{ + description: "a field starting with a number should be bracketed", + fieldArray: []string{"metadata", "fields", "46days"}, + expectedFieldName: "metadata.fields[46days]", + }) + tests = append(tests, testCase{ + description: "compound names should be bracketed", + fieldArray: []string{"metadata", "labels", "rancher.cattle.io/moo"}, + expectedFieldName: "metadata.labels[rancher.cattle.io/moo]", + }) + tests = append(tests, testCase{ + description: "space-separated names should be bracketed", + fieldArray: []string{"metadata", "labels", "space here"}, + expectedFieldName: "metadata.labels[space here]", + }) + tests = append(tests, testCase{ + description: "already-bracketed terms cause double-bracketing and should never be used", + fieldArray: []string{"metadata", "labels[k8s.io/deepcode]"}, + expectedFieldName: "metadata[labels[k8s.io/deepcode]]", + }) + tests = append(tests, testCase{ + description: "an empty array should be an empty string", + fieldArray: []string{}, + expectedFieldName: "", + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + gotFieldName := smartJoin(test.fieldArray) + assert.Equal(t, test.expectedFieldName, gotFieldName) + }) + } +} + +func TestBuildSortLabelsClause(t *testing.T) { + type testCase struct { + description string + labelName string + joinTableIndexByLabelName map[string]int + direction bool + expectedStmt string + expectedParam string + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "TestBuildSortClause: empty index list errors", + labelName: "emptyListError", + expectedErr: `internal error: no join-table index given for labelName "emptyListError"`, + }) + tests = append(tests, testCase{ + description: "TestBuildSortClause: hit ascending", + labelName: "testBSL1", + joinTableIndexByLabelName: map[string]int{"db:testBSL1": 3}, + direction: true, + expectedStmt: `(CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST`, + expectedParam: "testBSL1", + }) + tests = append(tests, testCase{ + description: "TestBuildSortClause: hit descending", + labelName: "testBSL2", + joinTableIndexByLabelName: map[string]int{"db:testBSL2": 4}, + direction: false, + expectedStmt: `(CASE lt4.label WHEN ? THEN lt4.value ELSE NULL END) DESC NULLS FIRST`, + expectedParam: "testBSL2", + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + stmt, param, err := buildSortLabelsClause(test.labelName, "db", test.joinTableIndexByLabelName, test.direction) + if test.expectedErr != "" { + assert.Equal(t, test.expectedErr, err.Error()) + } else { + assert.Nil(t, err) + assert.Equal(t, test.expectedStmt, stmt) + assert.Equal(t, test.expectedParam, param) + } + }) + } +} + +func TestConstructIndirectLabelFilterQuery(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.NotEq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.NotEq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" != ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Lt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"10"}, + Op: sqltypes.Lt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", float64(10)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Gt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"11"}, + Op: sqltypes.Gt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", float64(11)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Op: sqltypes.Exists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" != NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect Not-sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Op: sqltypes.NotExists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" == NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.In-Set", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"fish", "cows", "ships"}, + Op: sqltypes.In, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "fish", "cows", "ships"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.NotIn", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"balloons", "clubs", "cheese"}, + Op: sqltypes.NotIn, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" NOT IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "balloons", "clubs", "cheese"}, + expectedErr: "", + }) + // There's no easy way to safely parameterize column names, as opposed to values, + // so verify that the code generator checked them for whitelisted characters. + // Allow only [-a-zA-Z0-9$_\[\].]+ + tests = append(tests, testCase{ + description: "IndirectFilterQuery: verify the injected field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "name ; drop database marks ; select * from _v1_Namespace", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'name ; drop database marks ; select * from _v1_Namespace'", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + assert.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } + +} + +func TestConstructIndirectNonLabelFilterQuery(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.NotEq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.NotEq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" != ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Lt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"10"}, + Op: sqltypes.Lt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{float64(10)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Gt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"11"}, + Op: sqltypes.Gt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{float64(11)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Exists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" != NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect Not-sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.NotExists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" == NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.In-Set", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"fish", "cows", "ships"}, + Op: sqltypes.In, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"fish", "cows", "ships"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.NotIn", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"balloons", "clubs", "cheese"}, + Op: sqltypes.NotIn, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" NOT IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"balloons", "clubs", "cheese"}, + expectedErr: "", + }) + // There's no easy way to safely parameterize column names, as opposed to values, + // so verify that the code generator checked them for whitelisted characters. + // Allow only [-a-zA-Z0-9$_\[\].]+ + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: verify the injected external field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "name ; drop database marks ; select * from _v1_Namespace", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'name ; drop database marks ; select * from _v1_Namespace'", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: verify the injected selecting field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1 ; drop database thought-this-is=-checked"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "column is invalid [metadata[queryField1 ; drop database thought-this-is=-checked]]: supplied column is invalid", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructMixedLabelIndirect(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - one label, one redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "radio"}, + Matches: []string{"fish"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.displayName" = ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"radio", "fish", "System"}, + expectedErr: "", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructMixedMultiTypes(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - mix of label,non-label x direct,indirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "suitcase"}, + Matches: []string{"valid"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"sprint"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + { + Field: []string{"status", "queryField2"}, + Matches: []string{"moisture"}, + Op: sqltypes.Eq, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "green"}, + Matches: []string{"squalor"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"tournaments.cattle.io/v3", "Diary", "metadata.name", "spocks.brain"}, + }, + }, + }, + }}, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.name" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + JOIN "tournaments.cattle.io_v3_Diary_fields" ext4 ON lt3.value = ext4."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) + OR (ext2."spec.displayName" = ?) + OR (f."status.queryField2" = ?)) + AND (lt3.label = ? AND ext4."spocks.brain" = ?) + AND (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"suitcase", "valid", "sprint", "moisture", "green", "squalor"}, + expectedErr: "", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructLabelIndirectSort(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, no filters, happy path", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM ( + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, ext2."spec.clusterName" AS __ix_ext2_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE lt1.label = ? +UNION ALL + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, NULL AS __ix_ext2_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + WHERE o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1 + JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?) +) +WHERE FALSE + ORDER BY __ix_ext2_spec_clusterName ASC NULLS LAST, __ix_f_metadata_name ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, two filters, happy path", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "radio"}, + Matches: []string{"camels"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"tournaments.cattle.io/v3", "Capsule", "metadata.namespace", "spec.heights"}, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM ( + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, ext4."spec.clusterName" AS __ix_ext4_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "tournaments.cattle.io_v3_Capsule_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.namespace" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + JOIN "management.cattle.io_v3_Project_fields" ext4 ON lt3.value = ext4."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.heights" = ?)) AND (lt3.label = ?) +UNION ALL + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, NULL AS __ix_ext4_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "tournaments.cattle.io_v3_Capsule_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.namespace" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.heights" = ?)) AND (o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1 + JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt3i1 ON o1.key = lt3i1.key + WHERE lt3i1.label = ?)) +) +WHERE FALSE + ORDER BY __ix_ext4_spec_clusterName ASC NULLS LAST, __ix_f_metadata_name ASC`, + expectedStmtArgs: []any{"radio", "camels", "System", "field.cattle.io/projectId", "radio", "camels", "System", "field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "foo; drop database bobby1", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'foo; drop database bobby1'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "bar; drop database bobby2"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'bar; drop database bobby2'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, not enough indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 3", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, too many indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "little", "bobby-tables"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 5", + }) + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + }) + } +} + +func TestConstructSimpleNonLabelIndirectSort(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, no filters, no labels", + // Find all mcio.Projects that have the same metadata.name as the namespace, and sort by associated spec.clusterName + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (FALSE) + ORDER BY ext1."spec.clusterName" ASC NULLS LAST`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "foo; drop database bobby1", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'foo; drop database bobby1'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "bar; drop database bobby2"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'bar; drop database bobby2'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, not enough indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 3", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, too many indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "little", "bobby-tables"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 5", + }) + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + }) + } +} diff --git a/pkg/sqlcache/integration_test.go b/pkg/sqlcache/integration_test.go index 69183270..f88102e9 100644 --- a/pkg/sqlcache/integration_test.go +++ b/pkg/sqlcache/integration_test.go @@ -18,9 +18,9 @@ import ( "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/envtest" - "github.com/rancher/steve/pkg/sqlcache/informer" "github.com/rancher/steve/pkg/sqlcache/informer/factory" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" ) const testNamespace = "sql-test" @@ -107,8 +107,8 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { err = i.waitForCacheReady(configMapNames, testNamespace, cache) require.NoError(err) - orFiltersForFilters := func(filters ...informer.Filter) []informer.OrFilter { - return []informer.OrFilter{ + orFiltersForFilters := func(filters ...sqltypes.Filter) []sqltypes.OrFilter { + return []sqltypes.OrFilter{ { Filters: filters, }, @@ -116,85 +116,85 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { } tests := []struct { name string - filters []informer.OrFilter + filters []sqltypes.OrFilter wantNames []string }{ { name: "matches filter", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }), wantNames: []string{"matches-filter"}, }, { name: "partial matches filter", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }), wantNames: []string{"matches-filter", "partial-matches"}, }, { name: "no matches for filter with underscore as it is interpreted literally", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalu_"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }), wantNames: nil, }, { name: "no matches for filter with percent sign as it is interpreted literally", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalu%"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }), wantNames: nil, }, { name: "match with special characters", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"c%%l_value"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }), wantNames: []string{"special-character-matches"}, }, { name: "match with literal backslash character", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{`my\windows\path`}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }), wantNames: []string{"backslash-character-matches"}, }, { name: "not eq filter", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.NotEq, + Op: sqltypes.NotEq, Partial: false, }), wantNames: []string{"partial-matches", "not-matches-filter", "missing", "special-character-matches", "backslash-character-matches"}, }, { name: "partial not eq filter", - filters: orFiltersForFilters(informer.Filter{ + filters: orFiltersForFilters(sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.NotEq, + Op: sqltypes.NotEq, Partial: true, }), wantNames: []string{"not-matches-filter", "missing", "special-character-matches", "backslash-character-matches"}, @@ -202,16 +202,16 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { { name: "multiple or filters match", filters: orFiltersForFilters( - informer.Filter{ + sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, - informer.Filter{ + sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"notequal"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, ), @@ -220,16 +220,16 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { { name: "or filters on different fields", filters: orFiltersForFilters( - informer.Filter{ + sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, - informer.Filter{ + sqltypes.Filter{ Field: []string{`metadata`, `name`}, Matches: []string{"missing"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, ), @@ -237,23 +237,23 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { }, { name: "and filters, both must match", - filters: []informer.OrFilter{ + filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"somevalue"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{`metadata`, `name`}, Matches: []string{"matches-filter"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, @@ -264,10 +264,10 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { { name: "no matches", filters: orFiltersForFilters( - informer.Filter{ + sqltypes.Filter{ Field: []string{"metadata", "annotations", "somekey"}, Matches: []string{"valueNotRepresented"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, ), @@ -278,13 +278,13 @@ func (i *IntegrationSuite) TestSQLCacheFilters() { for _, test := range tests { test := test i.Run(test.name, func() { - options := informer.ListOptions{ + options := sqltypes.ListOptions{ Filters: test.filters, } partitions := []partition.Partition{defaultPartition} ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - cfgMaps, total, continueToken, err := cache.ListByOptions(ctx, options, partitions, testNamespace) + cfgMaps, total, continueToken, err := cache.ListByOptions(ctx, &options, partitions, testNamespace) i.Require().NoError(err) // since there's no additional pages, the continue token should be empty i.Require().Equal("", continueToken) @@ -334,11 +334,11 @@ func (i *IntegrationSuite) waitForCacheReady(readyResourceNames []string, namesp ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() return wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { - var options informer.ListOptions + var options sqltypes.ListOptions partitions := []partition.Partition{defaultPartition} cacheCtx, cacheCancel := context.WithTimeout(ctx, time.Second*5) defer cacheCancel() - currentResources, total, _, err := cache.ListByOptions(cacheCtx, options, partitions, namespace) + currentResources, total, _, err := cache.ListByOptions(cacheCtx, &options, partitions, namespace) if err != nil { // note that we don't return the error since that would stop the polling return false, nil diff --git a/pkg/sqlcache/informer/listoptions.go b/pkg/sqlcache/sqltypes/types.go similarity index 62% rename from pkg/sqlcache/informer/listoptions.go rename to pkg/sqlcache/sqltypes/types.go index 71bf6f6e..6fae8d7b 100644 --- a/pkg/sqlcache/informer/listoptions.go +++ b/pkg/sqlcache/sqltypes/types.go @@ -1,4 +1,4 @@ -package informer +package sqltypes type Op string @@ -25,25 +25,27 @@ const ( // ListOptions represents the query parameters that may be included in a list request. type ListOptions struct { - ChunkSize int - Resume string - Filters []OrFilter - Sort Sort - Pagination Pagination + ChunkSize int `json:"chunkSize"` + Resume string `json:"resume"` + Filters []OrFilter `json:"orFilters"` + SortList SortList `json:"sortList"` + Pagination Pagination `json:"pagination"` } // Filter represents a field to filter by. // A subfield in an object is represented in a request query using . notation, e.g. 'metadata.name'. // The subfield is internally represented as a slice, e.g. [metadata, name]. -// Complex subfields need to be expressed with square brackets, as in `metadata.labels[zombo.com/moose]`, -// but are mapped to the string slice ["metadata", "labels", "zombo.com/moose"] +// Complex subfields need to be expressed with square brackets, as in `metadata.labels[example.com/moose]`, +// but are mapped to the string slice ["metadata", "labels", "example.com/moose"] // // If more than one value is given for the `Match` field, we do an "IN ()" test type Filter struct { - Field []string - Matches []string - Op Op - Partial bool + Field []string `json:"fields"` + Matches []string `json:"matches"` + Op Op `json:"op"` + Partial bool `json:"partial"` + IsIndirect bool `json:"isIndirect"` + IndirectFields []string `json:"indirectFields"` } // OrFilter represents a set of possible fields to filter by, where an item may match any filter in the set to be included in the result. @@ -57,12 +59,24 @@ type OrFilter struct { // The order is represented by prefixing the sort key by '-', e.g. sort=-metadata.name. // e.g. To sort internal clusters first followed by clusters in alpha order: sort=-spec.internal,spec.displayName type Sort struct { - Fields [][]string - Orders []SortOrder + Fields []string `json:"fields"` + Order SortOrder `json:"order"` + IsIndirect bool `json:"isIndirect"` + IndirectFields []string `json:"indirectFields"` +} + +type SortList struct { + SortDirectives []Sort `json:"sortDirectives"` } // Pagination represents how to return paginated results. type Pagination struct { - PageSize int - Page int + PageSize int `json:"pageSize"` + Page int `json:"page"` +} + +func NewSortList() *SortList { + return &SortList{ + SortDirectives: []Sort{}, + } } diff --git a/pkg/stores/sqlpartition/listprocessor/processor.go b/pkg/stores/sqlpartition/listprocessor/processor.go index f1b89e7c..1e6813a3 100644 --- a/pkg/stores/sqlpartition/listprocessor/processor.go +++ b/pkg/stores/sqlpartition/listprocessor/processor.go @@ -3,6 +3,7 @@ package listprocessor import ( "context" + "errors" "fmt" "regexp" "strconv" @@ -10,8 +11,8 @@ import ( "github.com/rancher/apiserver/pkg/apierror" "github.com/rancher/apiserver/pkg/types" - "github.com/rancher/steve/pkg/sqlcache/informer" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/rancher/steve/pkg/stores/queryhelper" "github.com/rancher/steve/pkg/stores/sqlpartition/queryparser" "github.com/rancher/steve/pkg/stores/sqlpartition/selection" @@ -36,27 +37,18 @@ const ( ) var endsWithBracket = regexp.MustCompile(`^(.+)\[(.+)]$`) -var mapK8sOpToRancherOp = map[selection.Operator]informer.Op{ - selection.Equals: informer.Eq, - selection.DoubleEquals: informer.Eq, - selection.PartialEquals: informer.Eq, - selection.NotEquals: informer.NotEq, - selection.NotPartialEquals: informer.NotEq, - selection.In: informer.In, - selection.NotIn: informer.NotIn, - selection.Exists: informer.Exists, - selection.DoesNotExist: informer.NotExists, - selection.LessThan: informer.Lt, - selection.GreaterThan: informer.Gt, -} - -// ListOptions represents the query parameters that may be included in a list request. -type ListOptions struct { - ChunkSize int - Resume string - Filters []informer.OrFilter - Sort informer.Sort - Pagination informer.Pagination +var mapK8sOpToRancherOp = map[selection.Operator]sqltypes.Op{ + selection.Equals: sqltypes.Eq, + selection.DoubleEquals: sqltypes.Eq, + selection.PartialEquals: sqltypes.Eq, + selection.NotEquals: sqltypes.NotEq, + selection.NotPartialEquals: sqltypes.NotEq, + selection.In: sqltypes.In, + selection.NotIn: sqltypes.NotIn, + selection.Exists: sqltypes.Exists, + selection.DoesNotExist: sqltypes.NotExists, + selection.LessThan: sqltypes.Lt, + selection.GreaterThan: sqltypes.Gt, } type Cache interface { @@ -66,10 +58,10 @@ type Cache interface { // - the total number of resources (returned list might be a subset depending on pagination options in lo) // - a continue token, if there are more pages after the returned one // - an error instead of all of the above if anything went wrong - ListByOptions(ctx context.Context, lo informer.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) + ListByOptions(ctx context.Context, lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) } -func k8sOpToRancherOp(k8sOp selection.Operator) (informer.Op, bool, error) { +func k8sOpToRancherOp(k8sOp selection.Operator) (sqltypes.Op, bool, error) { v, ok := mapK8sOpToRancherOp[k8sOp] if ok { return v, k8sOp == selection.PartialEquals || k8sOp == selection.NotPartialEquals, nil @@ -77,21 +69,24 @@ func k8sOpToRancherOp(k8sOp selection.Operator) (informer.Op, bool, error) { return "", false, fmt.Errorf("unknown k8sOp: %s", k8sOp) } -func k8sRequirementToOrFilter(requirement queryparser.Requirement) (informer.Filter, error) { +func k8sRequirementToOrFilter(requirement queryparser.Requirement) (sqltypes.Filter, error) { values := requirement.Values() queryFields := splitQuery(requirement.Key()) op, usePartialMatch, err := k8sOpToRancherOp(requirement.Operator()) - return informer.Filter{ - Field: queryFields, - Matches: values, - Op: op, - Partial: usePartialMatch, + isIndirect, indirectFields := requirement.IndirectInfo() + return sqltypes.Filter{ + Field: queryFields, + Matches: values, + Op: op, + Partial: usePartialMatch, + IsIndirect: isIndirect, + IndirectFields: indirectFields, }, err } // ParseQuery parses the query params of a request and returns a ListOptions. -func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (informer.ListOptions, error) { - opts := informer.ListOptions{} +func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (sqltypes.ListOptions, error) { + opts := sqltypes.ListOptions{} opts.ChunkSize = getLimit(apiOp) @@ -100,13 +95,13 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (informer.ListOpt opts.Resume = cont filterParams := q[filterParam] - filterOpts := []informer.OrFilter{} + filterOpts := []sqltypes.OrFilter{} for _, filters := range filterParams { - requirements, err := queryparser.ParseToRequirements(filters) + requirements, err := queryparser.ParseToRequirements(filters, filterParam) if err != nil { - return informer.ListOptions{}, err + return sqltypes.ListOptions{}, err } - orFilter := informer.OrFilter{} + orFilter := sqltypes.OrFilter{} for _, requirement := range requirements { filter, err := k8sRequirementToOrFilter(requirement) if err != nil { @@ -118,29 +113,43 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (informer.ListOpt } opts.Filters = filterOpts - sortOpts := informer.Sort{} - sortKeys := q.Get(sortParam) - if sortKeys != "" { - sortParts := strings.Split(sortKeys, ",") - for _, sortPart := range sortParts { - field := sortPart - if len(field) > 0 { - sortOrder := informer.ASC - if field[0] == '-' { - sortOrder = informer.DESC - field = field[1:] - } - if len(field) > 0 { - sortOpts.Fields = append(sortOpts.Fields, queryhelper.SafeSplit(field)) - sortOpts.Orders = append(sortOpts.Orders, sortOrder) - } - } + if q.Has(sortParam) { + sortKeys := q.Get(sortParam) + filterRequirements, err := queryparser.ParseToRequirements(sortKeys, sortParam) + if err != nil { + return opts, err } + if len(filterRequirements) == 0 { + if len(sortKeys) == 0 { + return opts, errors.New("invalid sort key: ") + } + return opts, fmt.Errorf("invalid sort key: '%s'", sortKeys) + } + sortList := *sqltypes.NewSortList() + for _, requirement := range filterRequirements { + if requirement.Operator() != selection.Exists { + return opts, fmt.Errorf("sort directive %s can't contain operator (%s)", sortKeys, requirement.Operator()) + } + key := requirement.Key() + order := sqltypes.ASC + if key[0] == '-' { + order = sqltypes.DESC + key = key[1:] + } + isIndirect, indirectFields := requirement.IndirectInfo() + sortDirective := sqltypes.Sort{ + Fields: queryhelper.SafeSplit(key), + Order: order, + IsIndirect: isIndirect, + IndirectFields: indirectFields, + } + sortList.SortDirectives = append(sortList.SortDirectives, sortDirective) + } + opts.SortList = sortList } - opts.Sort = sortOpts var err error - pagination := informer.Pagination{} + pagination := sqltypes.Pagination{} pagination.PageSize, err = strconv.Atoi(q.Get(pageSizeParam)) if err != nil { pagination.PageSize = 0 @@ -151,12 +160,12 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (informer.ListOpt } opts.Pagination = pagination - op := informer.Eq + op := sqltypes.Eq projectsOrNamespaces := q.Get(projectsOrNamespacesVar) if projectsOrNamespaces == "" { projectsOrNamespaces = q.Get(projectsOrNamespacesVar + notOp) if projectsOrNamespaces != "" { - op = informer.NotEq + op = sqltypes.NotEq } } if projectsOrNamespaces != "" { @@ -167,12 +176,12 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (informer.ListOpt if projOrNSFilters == nil { return opts, apierror.NewAPIError(validation.NotFound, fmt.Sprintf("could not find any namespaces named [%s] or namespaces belonging to project named [%s]", projectsOrNamespaces, projectsOrNamespaces)) } - if op == informer.NotEq { + if op == sqltypes.NotEq { for _, filter := range projOrNSFilters { - opts.Filters = append(opts.Filters, informer.OrFilter{Filters: []informer.Filter{filter}}) + opts.Filters = append(opts.Filters, sqltypes.OrFilter{Filters: []sqltypes.Filter{filter}}) } } else { - opts.Filters = append(opts.Filters, informer.OrFilter{Filters: projOrNSFilters}) + opts.Filters = append(opts.Filters, sqltypes.OrFilter{Filters: projOrNSFilters}) } } @@ -205,22 +214,22 @@ func splitQuery(query string) []string { return strings.Split(query, ".") } -func parseNamespaceOrProjectFilters(ctx context.Context, projOrNS string, op informer.Op, namespaceInformer Cache) ([]informer.Filter, error) { - var filters []informer.Filter +func parseNamespaceOrProjectFilters(ctx context.Context, projOrNS string, op sqltypes.Op, namespaceInformer Cache) ([]sqltypes.Filter, error) { + var filters []sqltypes.Filter for _, pn := range strings.Split(projOrNS, ",") { - uList, _, _, err := namespaceInformer.ListByOptions(ctx, informer.ListOptions{ - Filters: []informer.OrFilter{ + uList, _, _, err := namespaceInformer.ListByOptions(ctx, &sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "name"}, Matches: []string{pn}, - Op: informer.Eq, + Op: sqltypes.Eq, }, { Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, Matches: []string{pn}, - Op: informer.Eq, + Op: sqltypes.Eq, }, }, }, @@ -230,7 +239,7 @@ func parseNamespaceOrProjectFilters(ctx context.Context, projOrNS string, op inf return filters, err } for _, item := range uList.Items { - filters = append(filters, informer.Filter{ + filters = append(filters, sqltypes.Filter{ Field: []string{"metadata", "namespace"}, Matches: []string{item.GetName()}, Op: op, diff --git a/pkg/stores/sqlpartition/listprocessor/processor_test.go b/pkg/stores/sqlpartition/listprocessor/processor_test.go index 0674f82b..92ba8af2 100644 --- a/pkg/stores/sqlpartition/listprocessor/processor_test.go +++ b/pkg/stores/sqlpartition/listprocessor/processor_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/rancher/apiserver/pkg/types" - "github.com/rancher/steve/pkg/sqlcache/informer" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -23,7 +23,7 @@ func TestParseQuery(t *testing.T) { setupNSCache func() Cache nsc Cache req *types.APIRequest - expectedLO informer.ListOptions + expectedLO sqltypes.ListOptions errExpected bool errorText string } @@ -35,10 +35,10 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: ""}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -51,21 +51,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "projectsornamespaces=somethin"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "namespace"}, Matches: []string{"ns1"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -82,19 +82,19 @@ func TestParseQuery(t *testing.T) { }, } nsc := NewMockCache(gomock.NewController(t)) - nsc.EXPECT().ListByOptions(context.Background(), informer.ListOptions{ - Filters: []informer.OrFilter{ + nsc.EXPECT().ListByOptions(context.Background(), &sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "name"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, { Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, }, }, @@ -111,40 +111,40 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "projectsornamespaces=somethin"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "namespace"}, Matches: []string{"ns1"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, errExpected: true, setupNSCache: func() Cache { nsi := NewMockCache(gomock.NewController(t)) - nsi.EXPECT().ListByOptions(context.Background(), informer.ListOptions{ - Filters: []informer.OrFilter{ + nsi.EXPECT().ListByOptions(context.Background(), &sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "name"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, { Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, }, }, @@ -161,21 +161,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "projectsornamespaces=somethin"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "namespace"}, Matches: []string{"ns1"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -185,19 +185,19 @@ func TestParseQuery(t *testing.T) { Items: []unstructured.Unstructured{}, } nsi := NewMockCache(gomock.NewController(t)) - nsi.EXPECT().ListByOptions(context.Background(), informer.ListOptions{ - Filters: []informer.OrFilter{ + nsi.EXPECT().ListByOptions(context.Background(), &sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "name"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, { Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, Matches: []string{"somethin"}, - Op: informer.Eq, + Op: sqltypes.Eq, }, }, }, @@ -213,21 +213,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a~c"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a"}, Matches: []string{"c"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -239,21 +239,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a=c"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a"}, Matches: []string{"c"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -274,21 +274,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=metadata.labels[grover.example.com/fish]~heads"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "labels", "grover.example.com/fish"}, Matches: []string{"heads"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -300,21 +300,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=metadata.annotations[chumley.example.com/fish]=seals"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "annotations", "chumley.example.com/fish"}, Matches: []string{"seals"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -326,20 +326,20 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=metadata.fields[3]<5"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "fields", "3"}, Matches: []string{"5"}, - Op: informer.Lt, + Op: sqltypes.Lt, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -351,21 +351,48 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=metadata.labels[grover.example.com/fish]~heads"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "labels", "grover.example.com/fish"}, Matches: []string{"heads"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ + Page: 1, + }, + }, + }) + tests = append(tests, testCase{ + description: "ParseQuery() with an indirect labels filter param should create an indirect labels-specific filter.", + req: &types.APIRequest{ + Request: &http.Request{ + URL: &url.URL{RawQuery: "filter=metadata.labels[grover.example.com/fish]=>[_v1][Foods][foodCode][country]=japan"}, + }, + }, + expectedLO: sqltypes.ListOptions{ + ChunkSize: defaultLimit, + Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "grover.example.com/fish"}, + Matches: []string{"japan"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"_v1", "Foods", "foodCode", "country"}, + }, + }, + }, + }, + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -377,31 +404,31 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a=c&filter=b=d"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a"}, Matches: []string{"c"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"b"}, Matches: []string{"d"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -413,31 +440,31 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a=c&filter=b=d"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a"}, Matches: []string{"c"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"b"}, Matches: []string{"d"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -449,27 +476,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=beer=pabst,metadata.labels[beer2.io/ale] ~schlitz"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"beer"}, Matches: []string{"pabst"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, { Field: []string{"metadata", "labels", "beer2.io/ale"}, Matches: []string{"schlitz"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -481,27 +508,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=beer=natty-bo,metadata.labels.beer3~rainier"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"beer"}, Matches: []string{"natty-bo"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: false, }, { Field: []string{"metadata", "labels", "beer3"}, Matches: []string{"rainier"}, - Op: informer.Eq, + Op: sqltypes.Eq, Partial: true, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -513,27 +540,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a1In in (x1),a2In IN (x2)"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a1In"}, Matches: []string{"x1"}, - Op: informer.In, + Op: sqltypes.In, Partial: false, }, { Field: []string{"a2In"}, Matches: []string{"x2"}, - Op: informer.In, + Op: sqltypes.In, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -545,21 +572,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a2In in (x2a, x2b)"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a2In"}, Matches: []string{"x2a", "x2b"}, - Op: informer.In, + Op: sqltypes.In, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -571,27 +598,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a1NotIn notin (x1),a2NotIn NOTIN (x2)"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a1NotIn"}, Matches: []string{"x1"}, - Op: informer.NotIn, + Op: sqltypes.NotIn, Partial: false, }, { Field: []string{"a2NotIn"}, Matches: []string{"x2"}, - Op: informer.NotIn, + Op: sqltypes.NotIn, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -603,21 +630,21 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a3NotIn in (x3a, x3b)"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a3NotIn"}, Matches: []string{"x3a", "x3b"}, - Op: informer.In, + Op: sqltypes.In, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -629,27 +656,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a4In iN (x4a),a4NotIn nOtIn (x4b)"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a4In"}, Matches: []string{"x4a"}, - Op: informer.In, + Op: sqltypes.In, Partial: false, }, { Field: []string{"a4NotIn"}, Matches: []string{"x4b"}, - Op: informer.NotIn, + Op: sqltypes.NotIn, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -671,33 +698,33 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=metadata.labels.a5In1,!metadata.labels.a5In2, ! metadata.labels.a5In3"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"metadata", "labels", "a5In1"}, - Op: informer.Exists, + Op: sqltypes.Exists, Matches: []string{}, Partial: false, }, { Field: []string{"metadata", "labels", "a5In2"}, - Op: informer.NotExists, + Op: sqltypes.NotExists, Matches: []string{}, Partial: false, }, { Field: []string{"metadata", "labels", "a5In3"}, - Op: informer.NotExists, + Op: sqltypes.NotExists, Matches: []string{}, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -709,27 +736,27 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "filter=a<1,b>2"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: []informer.OrFilter{ + Filters: []sqltypes.OrFilter{ { - Filters: []informer.Filter{ + Filters: []sqltypes.Filter{ { Field: []string{"a"}, - Op: informer.Lt, + Op: sqltypes.Lt, Matches: []string{"1"}, Partial: false, }, { Field: []string{"b"}, - Op: informer.Gt, + Op: sqltypes.Gt, Matches: []string{"2"}, Partial: false, }, }, }, }, - Pagination: informer.Pagination{ + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -742,15 +769,18 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "sort=metadata.name"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Sort: informer.Sort{ - Fields: [][]string{ - {"metadata", "name"}}, - Orders: []informer.SortOrder{informer.ASC}, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "name"}, + Order: sqltypes.ASC, + }, + }, }, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -763,14 +793,18 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "sort=-metadata.name"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Sort: informer.Sort{ - Fields: [][]string{{"metadata", "name"}}, - Orders: []informer.SortOrder{informer.DESC}, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "name"}, + Order: sqltypes.DESC, + }, + }, }, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -783,20 +817,22 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "sort=-metadata.name,spec.something"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Sort: informer.Sort{ - Fields: [][]string{ - {"metadata", "name"}, - {"spec", "something"}, - }, - Orders: []informer.SortOrder{ - informer.DESC, - informer.ASC, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "name"}, + Order: sqltypes.DESC, + }, + { + Fields: []string{"spec", "something"}, + Order: sqltypes.ASC, + }, }, }, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -809,17 +845,30 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "sort=-metadata.labels[beef.cattle.io/snort],metadata.labels.steer,metadata.labels[bossie.cattle.io/moo],spec.something"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Sort: informer.Sort{ - Fields: [][]string{{"metadata", "labels", "beef.cattle.io/snort"}, - {"metadata", "labels", "steer"}, - {"metadata", "labels", "bossie.cattle.io/moo"}, - {"spec", "something"}}, - Orders: []informer.SortOrder{informer.DESC, informer.ASC, informer.ASC, informer.ASC}, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "beef.cattle.io/snort"}, + Order: sqltypes.DESC, + }, + { + Fields: []string{"metadata", "labels", "steer"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"metadata", "labels", "bossie.cattle.io/moo"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"spec", "something"}, + Order: sqltypes.ASC, + }, + }, }, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -835,11 +884,11 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "continue=5"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, Resume: "5", - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -852,11 +901,11 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "continue=5"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, Resume: "5", - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -869,10 +918,10 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "limit=3"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: 3, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 1, }, }, @@ -885,10 +934,10 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "page=3"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ Page: 3, }, }, @@ -901,10 +950,10 @@ func TestParseQuery(t *testing.T) { URL: &url.URL{RawQuery: "pagesize=20"}, }, }, - expectedLO: informer.ListOptions{ + expectedLO: sqltypes.ListOptions{ ChunkSize: defaultLimit, - Filters: make([]informer.OrFilter, 0), - Pagination: informer.Pagination{ + Filters: make([]sqltypes.OrFilter, 0), + Pagination: sqltypes.Pagination{ PageSize: 20, Page: 1, }, diff --git a/pkg/stores/sqlpartition/listprocessor/proxy_mocks_test.go b/pkg/stores/sqlpartition/listprocessor/proxy_mocks_test.go index 06598043..22c057be 100644 --- a/pkg/stores/sqlpartition/listprocessor/proxy_mocks_test.go +++ b/pkg/stores/sqlpartition/listprocessor/proxy_mocks_test.go @@ -13,8 +13,8 @@ import ( context "context" reflect "reflect" - informer "github.com/rancher/steve/pkg/sqlcache/informer" partition "github.com/rancher/steve/pkg/sqlcache/partition" + sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes" gomock "go.uber.org/mock/gomock" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -43,7 +43,7 @@ func (m *MockCache) EXPECT() *MockCacheMockRecorder { } // ListByOptions mocks base method. -func (m *MockCache) ListByOptions(arg0 context.Context, arg1 informer.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { +func (m *MockCache) ListByOptions(arg0 context.Context, arg1 *sqltypes.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListByOptions", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*unstructured.UnstructuredList) diff --git a/pkg/stores/sqlpartition/queryparser/selector.go b/pkg/stores/sqlpartition/queryparser/selector.go index 5d67f738..73133226 100644 --- a/pkg/stores/sqlpartition/queryparser/selector.go +++ b/pkg/stores/sqlpartition/queryparser/selector.go @@ -39,6 +39,9 @@ the array into a sql statement. So the set gives us no benefit apart from removi 6. We allow `lt` and `gt` as aliases for `<` and `>`. 7. We added the '~' and '!~' operators to indicate partial match and non-match + +8. We added indirect field selection so we can base a filter or sort off a related value + (could be in a different table) */ package queryparser @@ -70,6 +73,7 @@ var ( string(selection.Equals), string(selection.DoubleEquals), string(selection.NotEquals), string(selection.PartialEquals), string(selection.NotPartialEquals), string(selection.GreaterThan), string(selection.LessThan), + string(selection.IndirectSelector), } validRequirementOperators = append(binaryOperators, unaryOperators...) labelSelectorRegex = regexp.MustCompile(`^metadata.labels(?:\.\w[-a-zA-Z0-9_./]*|\[.*])$`) @@ -135,7 +139,9 @@ type Requirement struct { // In huge majority of cases we have at most one value here. // It is generally faster to operate on a single-element slice // than on a single-element map, so we have a slice here. - strValues []string + strValues []string + isIndirect bool + indirectFields []string } // NewRequirement is the constructor for a Requirement. @@ -183,7 +189,26 @@ func NewRequirement(key string, op selection.Operator, vals []string, opts ...fi default: allErrs = append(allErrs, field.NotSupported(path.Child("operator"), op, validRequirementOperators)) } - return &Requirement{key: key, operator: op, strValues: vals}, allErrs.ToAggregate() + agg := allErrs.ToAggregate() + var err error + if agg != nil { + err = errors.New(agg.Error()) + } + return &Requirement{key: key, operator: op, strValues: vals}, err +} + +func NewIndirectRequirement(key string, indirectFields []string, newOperator *selection.Operator, targetValues []string, opts ...field.PathOption) (*Requirement, error) { + if newOperator == nil { + operator := selection.Exists + newOperator = &operator + } + r, err := NewRequirement(key, *newOperator, targetValues) + if err != nil { + return nil, err + } + r.isIndirect = true + r.indirectFields = indirectFields + return r, nil } func (r *Requirement) hasValue(value string) bool { @@ -214,6 +239,10 @@ func (r *Requirement) Values() []string { return ret.List() } +func (r *Requirement) IndirectInfo() (bool, []string) { + return r.isIndirect, r.indirectFields +} + // Equal checks the equality of requirement. func (r Requirement) Equal(x Requirement) bool { if r.key != x.key { @@ -377,6 +406,8 @@ const ( NotPartialEqualsToken // OpenParToken represents open parenthesis OpenParToken + // IndirectAccessToken is =>, used to associate one table with a related one, and grab a different field + IndirectAccessToken ) // string2token contains the mapping between lexer Token and token literal @@ -395,6 +426,7 @@ var string2token = map[string]Token{ "!~": NotPartialEqualsToken, "notin": NotInToken, "(": OpenParToken, + "=>": IndirectAccessToken, } // ScannedItem contains the Token and the literal produced by the lexer. @@ -405,7 +437,7 @@ type ScannedItem struct { func isIdentifierStartChar(ch byte) bool { r := rune(ch) - return unicode.IsLetter(r) || unicode.IsDigit(r) || ch == '_' + return unicode.IsLetter(r) || unicode.IsDigit(r) || ch == '_' || ch == '[' || ch == '-' } // isWhitespace returns true if the rune is a space, tab, or newline. @@ -531,6 +563,7 @@ type Parser struct { l *Lexer scannedItems []ScannedItem position int + parseType string path *field.Path } @@ -624,13 +657,18 @@ func (p *Parser) parseRequirement() (*Requirement, error) { if err != nil { return nil, err } + fieldPath := field.WithPath(p.path) if operator == selection.Exists || operator == selection.DoesNotExist { // operator found lookahead set checked - if !labelSelectorRegex.MatchString(key) { + if p.parseType == "filter" && !labelSelectorRegex.MatchString(key) { return nil, fmt.Errorf("existence tests are valid only for labels; not valid for field '%s'", key) } - return NewRequirement(key, operator, []string{}, field.WithPath(p.path)) + return NewRequirement(key, operator, []string{}, fieldPath) } - operator, err = p.parseOperator() + return p.parseOperatorAndValues(key, fieldPath, true) +} + +func (p *Parser) parseOperatorAndValues(key string, fieldPath field.PathOption, allowIndirectSelector bool) (*Requirement, error) { + operator, err := p.parseOperator() if err != nil { return nil, err } @@ -640,12 +678,22 @@ func (p *Parser) parseRequirement() (*Requirement, error) { values, err = p.parseValues() case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.GreaterThan, selection.LessThan, selection.PartialEquals, selection.NotPartialEquals: values, err = p.parseSingleValue() + case selection.IndirectSelector: + if !allowIndirectSelector { + return nil, fmt.Errorf("found a subsequent indirect selector (->)") + } + indirectFields, newOperator, targetValues, err := p.parseIndirectAccessorPart(key, fieldPath) + if err != nil { + return nil, err + } else if newOperator != nil && p.parseType == "sort" { + return nil, fmt.Errorf("found an operator (%s) in a sort expression )", *newOperator) + } + return NewIndirectRequirement(key, indirectFields, newOperator, targetValues.List(), fieldPath) } if err != nil { return nil, err } return NewRequirement(key, operator, values.List(), field.WithPath(p.path)) - } // parseKeyAndInferOperator parses literals. @@ -694,11 +742,15 @@ func (p *Parser) parseOperator() (op selection.Operator, err error) { op = selection.NotEquals case NotPartialEqualsToken: op = selection.NotPartialEquals + case IndirectAccessToken: + op = selection.IndirectSelector default: if lit == "lt" { op = selection.LessThan } else if lit == "gt" { op = selection.GreaterThan + } else if p.parseType == "sort" { + return "", fmt.Errorf("found unexpected token '%s' in sort parameter", lit) } else { return "", fmt.Errorf("found '%s', expected: %v", lit, strings.Join(binaryOperators, ", ")) } @@ -727,10 +779,38 @@ func (p *Parser) parseValues() (sets.String, error) { p.consume(Values) return sets.NewString(""), nil default: - return nil, fmt.Errorf("found '%s', expected: ',', ')' or identifier", lit) + return sets.NewString(""), fmt.Errorf("found '%s', expected: ',', ')' or identifier", lit) } } +func (p *Parser) parseIndirectAccessorPart(key string, fieldPath field.PathOption) ([]string, *selection.Operator, sets.String, error) { + //key string, indirectFields []string, newOperator selection.Operator, targetValues []string + values := sets.String{} + tok, lit := p.consume(Values) + if tok != IdentifierToken { + return nil, nil, values, fmt.Errorf("found '%s', expected: an indirect field specifier", lit) + } + matched, err := regexp.MatchString(`^(?:\[.*?\])+$`, lit) + if err != nil { + return nil, nil, values, err + } else if !matched { + return nil, nil, values, fmt.Errorf("found '%s', expected: a sequence of bracketed identifiers", lit) + } + + indirectFields := strings.Split(lit[1:len(lit)-1], "][") + if len(indirectFields) != 4 { + return nil, nil, values, fmt.Errorf("found '%s', expected: a sequence of three bracketed identifiers", lit) + } + if p.parseType == "sort" { + return indirectFields, nil, sets.NewString(), nil + } + r, err := p.parseOperatorAndValues(key, fieldPath, false) + if err != nil { + return nil, nil, values, err + } + return indirectFields, &r.operator, sets.NewString(r.strValues...), nil +} + // parseIdentifiersList parses a (possibly empty) list of // of comma separated (possibly empty) identifiers func (p *Parser) parseIdentifiersList() (sets.String, error) { @@ -814,9 +894,9 @@ func (p *Parser) parseSingleValue() (sets.String, error) { // 4. A requirement with just a KEY - as in "y" above - denotes that // the KEY exists and can be any VALUE. // 5. A requirement with just !KEY requires that the KEY not exist. -func Parse(selector string, opts ...field.PathOption) (Selector, error) { +func Parse(selector string, parseType string, opts ...field.PathOption) (Selector, error) { pathThing := field.ToPath(opts...) - parsedSelector, err := parse(selector, pathThing) + parsedSelector, err := parse(selector, parseType, pathThing) if err == nil { return parsedSelector, nil } @@ -827,8 +907,8 @@ func Parse(selector string, opts ...field.PathOption) (Selector, error) { // The callers of this method can then decide how to return the internalSelector struct to their // callers. This function has two callers now, one returns a Selector interface and the other // returns a list of requirements. -func parse(selector string, path *field.Path) (internalSelector, error) { - p := &Parser{l: &Lexer{s: selector, pos: 0}, path: path} +func parse(selector string, parseType string, path *field.Path) (internalSelector, error) { + p := &Parser{l: &Lexer{s: selector, pos: 0}, parseType: parseType, path: path} items, err := p.parse() if err != nil { return nil, err @@ -883,8 +963,8 @@ func SelectorFromValidatedSet(ls Set) Selector { // processing on selector requirements. // See the documentation for Parse() function for more details. // TODO: Consider exporting the internalSelector type instead. -func ParseToRequirements(selector string, opts ...field.PathOption) ([]Requirement, error) { - return parse(selector, field.ToPath(opts...)) +func ParseToRequirements(selector string, parseType string, opts ...field.PathOption) ([]Requirement, error) { + return parse(selector, parseType, field.ToPath(opts...)) } // ValidatedSetSelector wraps a Set, allowing it to implement the Selector interface. Unlike diff --git a/pkg/stores/sqlpartition/queryparser/selector_test.go b/pkg/stores/sqlpartition/queryparser/selector_test.go index f11b3c1a..87ed323c 100644 --- a/pkg/stores/sqlpartition/queryparser/selector_test.go +++ b/pkg/stores/sqlpartition/queryparser/selector_test.go @@ -27,15 +27,9 @@ import ( "strings" "testing" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/rancher/steve/pkg/stores/sqlpartition/selection" + "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -var ( - ignoreDetail = cmpopts.IgnoreFields(field.Error{}, "Detail") ) func TestSelectorParse(t *testing.T) { @@ -54,6 +48,9 @@ func TestSelectorParse(t *testing.T) { "metadata.labels[im.here]", "!metadata.labels[im.not.here]", "metadata.labels[k8s.io/meta-stuff] ~ has-dashes_underscores.dots.only", + "metadata.labels[k8s.io/meta-stuff] => [management.cattle.io/v3][tokens][id][metadata.state.name] = active", + "name => [management.cattle.io/v3][tokens][id][metadata.state.name] = active", + "metadata.annotations[blah] => [management.cattle.io/v3][tokens][id][metadata.state.name] = active", } testBadStrings := []string{ "!no-label-absence-test", @@ -77,15 +74,22 @@ func TestSelectorParse(t *testing.T) { "!metadata.labels(im.not.here)", `x="no double quotes allowed"`, `x='no single quotes allowed'`, + "metadata.labels[k8s.io/meta-stuff] => not-bracketed = active", + "metadata.labels[k8s.io/meta-stuff] => [not][enough][accessors] = active", + "metadata.labels[k8s.io/meta-stuff] => [too][many][accessors][by][1] = active", + "metadata.labels[k8s.io/meta-stuff] => [missing][an][operator][end-of-string]", + "metadata.labels[k8s.io/meta-stuff] => [missing][an][operator][no-following-operator] no-operator", + "metadata.labels[k8s.io/meta-stuff] => [missing][a][post-operator][value] >", + "metadata.labels[not/followed/by/accessor] => = active", } for _, test := range testGoodStrings { - _, err := Parse(test) + _, err := Parse(test, "filter") if err != nil { t.Errorf("%v: error %v (%#v)\n", test, err, err) } } for _, test := range testBadStrings { - _, err := Parse(test) + _, err := Parse(test, "filter") if err == nil { t.Errorf("%v: did not get expected error\n", test) } @@ -115,6 +119,7 @@ func TestLexer(t *testing.T) { {"~", PartialEqualsToken}, {"!~", NotPartialEqualsToken}, {"||", ErrorToken}, + {"=>", IndirectAccessToken}, } for _, v := range testcases { l := &Lexer{s: v.s, pos: 0} @@ -163,6 +168,9 @@ func TestLexerSequence(t *testing.T) { {"key!~ value", []Token{IdentifierToken, NotPartialEqualsToken, IdentifierToken}}, {"key !~value", []Token{IdentifierToken, NotPartialEqualsToken, IdentifierToken}}, {"key!~value", []Token{IdentifierToken, NotPartialEqualsToken, IdentifierToken}}, + {"metadata.labels[k8s.io/meta-stuff] => [management.cattle.io/v3][tokens][id][metadata.state.name] = active", + []Token{IdentifierToken, IndirectAccessToken, IdentifierToken, EqualsToken, IdentifierToken}, + }, } for _, v := range testcases { var tokens []Token @@ -203,6 +211,10 @@ func TestParserLookahead(t *testing.T) { {"key gt 3", []Token{IdentifierToken, GreaterThanToken, IdentifierToken, EndOfStringToken}}, {"key lt 4", []Token{IdentifierToken, LessThanToken, IdentifierToken, EndOfStringToken}}, {`key = multi-word-string`, []Token{IdentifierToken, EqualsToken, QuotedStringToken, EndOfStringToken}}, + + {"metadata.labels[k8s.io/meta-stuff] => [management.cattle.io/v3][tokens][id][metadata.state.name] = active", + []Token{IdentifierToken, IndirectAccessToken, IdentifierToken, EqualsToken, IdentifierToken, EndOfStringToken}, + }, } for _, v := range testcases { p := &Parser{l: &Lexer{s: v.s, pos: 0}, position: 0} @@ -240,6 +252,7 @@ func TestParseOperator(t *testing.T) { {"notin", nil}, {"!=", nil}, {"!~", nil}, + {"=>", nil}, {"!", fmt.Errorf("found '%s', expected: %v", selection.DoesNotExist, strings.Join(binaryOperators, ", "))}, {"exists", fmt.Errorf("found '%s', expected: %v", selection.Exists, strings.Join(binaryOperators, ", "))}, {"(", fmt.Errorf("found '%s', expected: %v", "(", strings.Join(binaryOperators, ", "))}, @@ -262,30 +275,18 @@ func TestRequirementConstructor(t *testing.T) { Key string Op selection.Operator Vals sets.String - WantErr field.ErrorList + WantErr string }{ { - Key: "x1", - Op: selection.In, - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values", - BadValue: []string{}, - }, - }, + Key: "x1", + Op: selection.In, + WantErr: "values: Invalid value: []string{}: for 'in', 'notin' operators, values set can't be empty", }, { - Key: "x2", - Op: selection.NotIn, - Vals: sets.NewString(), - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values", - BadValue: []string{}, - }, - }, + Key: "x2", + Op: selection.NotIn, + Vals: sets.NewString(), + WantErr: "values: Invalid value: []string{}: for 'in', 'notin' operators, values set can't be empty", }, { Key: "x3", @@ -298,16 +299,10 @@ func TestRequirementConstructor(t *testing.T) { Vals: sets.NewString("foo"), }, { - Key: "x5", - Op: selection.Equals, - Vals: sets.NewString("foo", "bar"), - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values", - BadValue: []string{"bar", "foo"}, - }, - }, + Key: "x5", + Op: selection.Equals, + Vals: sets.NewString("foo", "bar"), + WantErr: "values: Invalid value: []string{\"bar\", \"foo\"}: exact-match compatibility requires one single value", }, { Key: "x6", @@ -318,16 +313,10 @@ func TestRequirementConstructor(t *testing.T) { Op: selection.DoesNotExist, }, { - Key: "x8", - Op: selection.Exists, - Vals: sets.NewString("foo"), - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values", - BadValue: []string{"foo"}, - }, - }, + Key: "x8", + Op: selection.Exists, + Vals: sets.NewString("foo"), + WantErr: `values: Invalid value: []string{"foo"}: values set must be empty for exists and does not exist`, }, { Key: "x9", @@ -350,39 +339,21 @@ func TestRequirementConstructor(t *testing.T) { Vals: sets.NewString("6"), }, { - Key: "x13", - Op: selection.GreaterThan, - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values", - BadValue: []string{}, - }, - }, + Key: "x13", + Op: selection.GreaterThan, + WantErr: "values: Invalid value: []string{}: for 'Gt', 'Lt' operators, exactly one value is required", }, { - Key: "x14", - Op: selection.GreaterThan, - Vals: sets.NewString("bar"), - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values[0]", - BadValue: "bar", - }, - }, + Key: "x14", + Op: selection.GreaterThan, + Vals: sets.NewString("bar"), + WantErr: `values[0]: Invalid value: "bar": for 'Gt', 'Lt' operators, the value must be an integer`, }, { - Key: "x15", - Op: selection.LessThan, - Vals: sets.NewString("bar"), - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "values[0]", - BadValue: "bar", - }, - }, + Key: "x15", + Op: selection.LessThan, + Vals: sets.NewString("bar"), + WantErr: `values[0]: Invalid value: "bar": for 'Gt', 'Lt' operators, the value must be an integer`, }, { Key: strings.Repeat("a", 254), //breaks DNS rule that len(key) <= 253 @@ -399,21 +370,29 @@ func TestRequirementConstructor(t *testing.T) { Vals: sets.NewString("a b"), }, { - Key: "x18", - Op: "unsupportedOp", - WantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeNotSupported, - Field: "operator", - BadValue: selection.Operator("unsupportedOp"), - }, - }, + Key: "x18", + Op: "unsupportedOp", + WantErr: `operator: Unsupported value: "unsupportedOp": supported values: "in", "notin", "=", "==", "!=", "~", "!~", "gt", "lt", "=>", "exists", "!"`, }, } for _, rc := range requirementConstructorTests { _, err := NewRequirement(rc.Key, rc.Op, rc.Vals.List()) - if diff := cmp.Diff(rc.WantErr.ToAggregate(), err, ignoreDetail); diff != "" { - t.Errorf("NewRequirement test %v returned unexpected error (-want,+got):\n%s", rc.Key, diff) + if rc.WantErr != "" { + assert.NotNil(t, err) + if err != nil { + assert.Equal(t, rc.WantErr, err.Error()) + } + } else { + assert.Nil(t, err) + } + _, err = NewIndirectRequirement(rc.Key, []string{"herb", "job", "nice", "reading"}, &rc.Op, rc.Vals.List()) + if rc.WantErr != "" { + assert.NotNil(t, err) + if err != nil { + assert.Equal(t, rc.WantErr, err.Error()) + } + } else { + assert.Nil(t, err) } } } diff --git a/pkg/stores/sqlpartition/selection/operator.go b/pkg/stores/sqlpartition/selection/operator.go index a0d0254b..bd7f2b06 100644 --- a/pkg/stores/sqlpartition/selection/operator.go +++ b/pkg/stores/sqlpartition/selection/operator.go @@ -38,4 +38,5 @@ const ( Exists Operator = "exists" GreaterThan Operator = "gt" LessThan Operator = "lt" + IndirectSelector Operator = "=>" ) diff --git a/pkg/stores/sqlproxy/proxy_mocks_test.go b/pkg/stores/sqlproxy/proxy_mocks_test.go index 3aeee7e5..e03065c9 100644 --- a/pkg/stores/sqlproxy/proxy_mocks_test.go +++ b/pkg/stores/sqlproxy/proxy_mocks_test.go @@ -14,9 +14,9 @@ import ( reflect "reflect" types "github.com/rancher/apiserver/pkg/types" - informer "github.com/rancher/steve/pkg/sqlcache/informer" factory "github.com/rancher/steve/pkg/sqlcache/informer/factory" partition "github.com/rancher/steve/pkg/sqlcache/partition" + sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes" summary "github.com/rancher/wrangler/v3/pkg/summary" gomock "go.uber.org/mock/gomock" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -51,7 +51,7 @@ func (m *MockCache) EXPECT() *MockCacheMockRecorder { } // ListByOptions mocks base method. -func (m *MockCache) ListByOptions(arg0 context.Context, arg1 informer.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { +func (m *MockCache) ListByOptions(arg0 context.Context, arg1 *sqltypes.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListByOptions", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*unstructured.UnstructuredList) diff --git a/pkg/stores/sqlproxy/proxy_store.go b/pkg/stores/sqlproxy/proxy_store.go index 96e03986..528996e8 100644 --- a/pkg/stores/sqlproxy/proxy_store.go +++ b/pkg/stores/sqlproxy/proxy_store.go @@ -36,6 +36,7 @@ import ( "github.com/rancher/steve/pkg/sqlcache/informer" "github.com/rancher/steve/pkg/sqlcache/informer/factory" "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/rancher/wrangler/v3/pkg/data" "github.com/rancher/wrangler/v3/pkg/schemas" "github.com/rancher/wrangler/v3/pkg/schemas/validation" @@ -216,7 +217,7 @@ type Cache interface { // - the total number of resources (returned list might be a subset depending on pagination options in lo) // - a continue token, if there are more pages after the returned one // - an error instead of all of the above if anything went wrong - ListByOptions(ctx context.Context, lo informer.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) + ListByOptions(ctx context.Context, lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) } // WarningBuffer holds warnings that may be returned from the kubernetes api @@ -783,7 +784,7 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, schema *types.APISchem return nil, 0, "", err } - list, total, continueToken, err := inf.ListByOptions(apiOp.Context(), opts, partitions, apiOp.Namespace) + list, total, continueToken, err := inf.ListByOptions(apiOp.Context(), &opts, partitions, apiOp.Namespace) if err != nil { if errors.Is(err, informer.ErrInvalidColumn) { return nil, 0, "", apierror.NewAPIError(validation.InvalidBodyContent, err.Error()) diff --git a/pkg/stores/sqlproxy/proxy_store_test.go b/pkg/stores/sqlproxy/proxy_store_test.go index a6f28d59..85cdb4ea 100644 --- a/pkg/stores/sqlproxy/proxy_store_test.go +++ b/pkg/stores/sqlproxy/proxy_store_test.go @@ -244,7 +244,7 @@ func TestListByPartitions(t *testing.T) { // This tests that fields are being extracted from schema columns and the type specific fields map cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) - bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) + bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) assert.Nil(t, err) assert.Equal(t, expectedItems, list) @@ -461,7 +461,7 @@ func TestListByPartitions(t *testing.T) { cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) - bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) + bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) assert.Nil(t, err) assert.Equal(t, expectedItems, list) @@ -610,7 +610,7 @@ func TestListByPartitions(t *testing.T) { cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) - bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error")) + bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error")) tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) _, _, _, err = s.ListByPartitions(req, schema, partitions) diff --git a/pkg/stores/sqlproxy/sql_informer_mocks_test.go b/pkg/stores/sqlproxy/sql_informer_mocks_test.go index 125f2192..ca237f8c 100644 --- a/pkg/stores/sqlproxy/sql_informer_mocks_test.go +++ b/pkg/stores/sqlproxy/sql_informer_mocks_test.go @@ -13,8 +13,8 @@ import ( context "context" reflect "reflect" - informer "github.com/rancher/steve/pkg/sqlcache/informer" partition "github.com/rancher/steve/pkg/sqlcache/partition" + sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes" gomock "go.uber.org/mock/gomock" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -43,7 +43,7 @@ func (m *MockByOptionsLister) EXPECT() *MockByOptionsListerMockRecorder { } // ListByOptions mocks base method. -func (m *MockByOptionsLister) ListByOptions(arg0 context.Context, arg1 informer.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { +func (m *MockByOptionsLister) ListByOptions(arg0 context.Context, arg1 *sqltypes.ListOptions, arg2 []partition.Partition, arg3 string) (*unstructured.UnstructuredList, int, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListByOptions", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*unstructured.UnstructuredList)