diff --git a/pkg/sqlcache/informer/listoption_indexer.go b/pkg/sqlcache/informer/listoption_indexer.go index 6aec0112..262ef346 100644 --- a/pkg/sqlcache/informer/listoption_indexer.go +++ b/pkg/sqlcache/informer/listoption_indexer.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "regexp" - "sort" "strconv" "strings" @@ -253,6 +252,8 @@ func (l *ListOptionIndexer) ListByOptions(ctx context.Context, lo *sqltypes.List if err != nil { return nil, 0, "", err } + logrus.Debugf("ListOptionIndexer prepared statement: %v", queryInfo.query) + logrus.Debugf("Params: %v", queryInfo.params) return l.executeQuery(ctx, queryInfo) } @@ -267,210 +268,6 @@ type QueryInfo struct { offset int } -func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) { - ensureSortLabelsAreSelected(lo) - queryInfo := &QueryInfo{} - queryUsesLabels := hasLabelFilter(lo.Filters) - joinTableIndexByLabelName := make(map[string]int) - - // First, what kind of filtering will we be doing? - // 1- Intro: SELECT and JOIN clauses - // There's a 1:1 correspondence between a base table and its _Fields table - // but it's possible that a key has no associated labels, so if we're doing a - // non-existence test on labels we need to do a LEFT OUTER JOIN - distinctModifier := "" - if queryUsesLabels { - distinctModifier = " DISTINCT" - } - query := fmt.Sprintf(`SELECT%s o.object, o.objectnonce, o.dekid FROM "%s" o`, distinctModifier, dbName) - query += "\n " - query += fmt.Sprintf(`JOIN "%s_fields" f ON o.key = f.key`, dbName) - if queryUsesLabels { - for i, orFilter := range lo.Filters { - for j, filter := range orFilter.Filters { - if isLabelFilter(&filter) { - labelName := filter.Field[2] - _, ok := joinTableIndexByLabelName[labelName] - if !ok { - // Make the lt index 1-based for readability - jtIndex := i + j + 1 - joinTableIndexByLabelName[labelName] = jtIndex - query += "\n " - query += fmt.Sprintf(`LEFT OUTER JOIN "%s_labels" lt%d ON o.key = lt%d.key`, dbName, jtIndex, jtIndex) - } - } - } - } - } - params := []any{} - - // 2- Filtering: WHERE clauses (from lo.Filters) - whereClauses := []string{} - for _, orFilters := range lo.Filters { - orClause, orParams, err := l.buildORClauseFromFilters(orFilters, dbName, joinTableIndexByLabelName) - if err != nil { - return queryInfo, err - } - if orClause == "" { - continue - } - whereClauses = append(whereClauses, orClause) - params = append(params, orParams...) - } - - // WHERE clauses (from namespace) - if namespace != "" && namespace != "*" { - whereClauses = append(whereClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) - params = append(params, namespace) - } - - // WHERE clauses (from partitions and their corresponding parameters) - partitionClauses := []string{} - for _, thisPartition := range partitions { - if thisPartition.Passthrough { - // nothing to do, no extra filtering to apply by definition - } else { - singlePartitionClauses := []string{} - - // filter by namespace - if thisPartition.Namespace != "" && thisPartition.Namespace != "*" { - singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) - params = append(params, thisPartition.Namespace) - } - - // optionally filter by names - if !thisPartition.All { - names := thisPartition.Names - - if len(names) == 0 { - // degenerate case, there will be no results - singlePartitionClauses = append(singlePartitionClauses, "FALSE") - } else { - singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.name" IN (?%s)`, strings.Repeat(", ?", len(thisPartition.Names)-1))) - // sort for reproducibility - sortedNames := thisPartition.Names.UnsortedList() - sort.Strings(sortedNames) - for _, name := range sortedNames { - params = append(params, name) - } - } - } - - if len(singlePartitionClauses) > 0 { - partitionClauses = append(partitionClauses, strings.Join(singlePartitionClauses, " AND ")) - } - } - } - if len(partitions) == 0 { - // degenerate case, there will be no results - whereClauses = append(whereClauses, "FALSE") - } - if len(partitionClauses) == 1 { - whereClauses = append(whereClauses, partitionClauses[0]) - } - if len(partitionClauses) > 1 { - whereClauses = append(whereClauses, "(\n ("+strings.Join(partitionClauses, ") OR\n (")+")\n)") - } - - if len(whereClauses) > 0 { - query += "\n WHERE\n " - for index, clause := range whereClauses { - query += fmt.Sprintf("(%s)", clause) - if index == len(whereClauses)-1 { - break - } - query += " AND\n " - } - } - - // before proceeding, save a copy of the query and params without LIMIT/OFFSET/ORDER info - // for COUNTing all results later - countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", query) - countParams := params[:] - - // 3- Sorting: ORDER BY clauses (from lo.Sort) - if len(lo.SortList.SortDirectives) > 0 { - orderByClauses := []string{} - for _, sortDirective := range lo.SortList.SortDirectives { - fields := sortDirective.Fields - if isLabelsFieldList(fields) { - clause, sortParam, err := buildSortLabelsClause(fields[2], joinTableIndexByLabelName, sortDirective.Order == sqltypes.ASC) - if err != nil { - return nil, err - } - orderByClauses = append(orderByClauses, clause) - params = append(params, sortParam) - } else { - columnName := toColumnName(fields) - if err := l.validateColumn(columnName); err != nil { - return queryInfo, err - } - direction := "ASC" - if sortDirective.Order == sqltypes.DESC { - direction = "DESC" - } - orderByClauses = append(orderByClauses, fmt.Sprintf(`f."%s" %s`, columnName, direction)) - } - } - query += "\n ORDER BY " - query += strings.Join(orderByClauses, ", ") - } else { - // make sure one default order is always picked - if l.namespaced { - query += "\n ORDER BY f.\"metadata.namespace\" ASC, f.\"metadata.name\" ASC " - } else { - query += "\n ORDER BY f.\"metadata.name\" ASC " - } - } - - // 4- Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume) - - limitClause := "" - // take the smallest limit between lo.Pagination and lo.ChunkSize - limit := lo.Pagination.PageSize - if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) { - limit = lo.ChunkSize - } - if limit > 0 { - limitClause = "\n LIMIT ?" - params = append(params, limit) - } - - // OFFSET clause (from lo.Pagination and/or lo.Resume) - offsetClause := "" - offset := 0 - if lo.Resume != "" { - offsetInt, err := strconv.Atoi(lo.Resume) - if err != nil { - return queryInfo, err - } - offset = offsetInt - } - if lo.Pagination.Page >= 1 { - offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1) - } - if offset > 0 { - offsetClause = "\n OFFSET ?" - params = append(params, offset) - } - if limit > 0 || offset > 0 { - query += limitClause - query += offsetClause - queryInfo.countQuery = countQuery - queryInfo.countParams = countParams - queryInfo.limit = limit - queryInfo.offset = offset - } - // Otherwise leave these as default values and the executor won't do pagination work - - logrus.Debugf("ListOptionIndexer prepared statement: %v", query) - logrus.Debugf("Params: %v", params) - queryInfo.query = query - queryInfo.params = params - - return queryInfo, nil -} - func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryInfo) (result *unstructured.UnstructuredList, total int, token string, err error) { stmt := l.Prepare(queryInfo.query) defer func() { @@ -528,338 +325,12 @@ func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryIn return toUnstructuredList(items), total, continueToken, nil } -func (l *ListOptionIndexer) validateColumn(column string) error { - for _, v := range l.indexedFields { - if v == column { - return nil - } +func extractSubFields(fields string) []string { + subfields := make([]string, 0) + for _, subField := range subfieldRegex.FindAllString(fields, -1) { + subfields = append(subfields, strings.TrimSuffix(subField, ".")) } - return fmt.Errorf("column is invalid [%s]: %w", column, ErrInvalidColumn) -} - -// buildORClause creates an SQLite compatible query that ORs conditions built from passed filters -func (l *ListOptionIndexer) buildORClauseFromFilters(orFilters sqltypes.OrFilter, dbName string, joinTableIndexByLabelName map[string]int) (string, []any, error) { - var params []any - clauses := make([]string, 0, len(orFilters.Filters)) - var newParams []any - var newClause string - var err error - - for _, filter := range orFilters.Filters { - if isLabelFilter(&filter) { - index, ok := joinTableIndexByLabelName[filter.Field[2]] - if !ok { - return "", nil, fmt.Errorf("internal error: no index for label name %s", filter.Field[2]) - } - newClause, newParams, err = l.getLabelFilter(index, filter, dbName) - } else { - newClause, newParams, err = l.getFieldFilter(filter) - } - if err != nil { - return "", nil, err - } - clauses = append(clauses, newClause) - params = append(params, newParams...) - } - switch len(clauses) { - case 0: - return "", params, nil - case 1: - return clauses[0], params, nil - } - return fmt.Sprintf("(%s)", strings.Join(clauses, ") OR (")), params, nil -} - -func buildSortLabelsClause(labelName string, joinTableIndexByLabelName map[string]int, isAsc bool) (string, string, error) { - ltIndex, ok := joinTableIndexByLabelName[labelName] - if !ok { - return "", "", fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) - } - stmt := fmt.Sprintf(`CASE lt%d.label WHEN ? THEN lt%d.value ELSE NULL END`, ltIndex, ltIndex) - dir := "ASC" - nullsPosition := "LAST" - if !isAsc { - dir = "DESC" - nullsPosition = "FIRST" - } - return fmt.Sprintf("(%s) %s NULLS %s", stmt, dir, nullsPosition), labelName, nil -} - -// If the user tries to sort on a particular label without mentioning it in a query, -// it turns out that the sort-directive is ignored. It could be that the sqlite engine -// is doing some kind of optimization on the `select distinct`, but verifying an otherwise -// unreferenced label exists solves this problem. -// And it's better to do this by modifying the ListOptions object. -// There are no thread-safety issues in doing this because the ListOptions object is -// created in Store.ListByPartitions, and that ends up calling ListOptionIndexer.ConstructQuery. -// No other goroutines access this object. -func ensureSortLabelsAreSelected(lo *sqltypes.ListOptions) { - if len(lo.SortList.SortDirectives) == 0 { - return - } - unboundSortLabels := make(map[string]bool) - for _, sortDirective := range lo.SortList.SortDirectives { - fields := sortDirective.Fields - if isLabelsFieldList(fields) { - unboundSortLabels[fields[2]] = true - } - } - if len(unboundSortLabels) == 0 { - return - } - // If we have sort directives but no filters, add an exists-filter for each label. - if lo.Filters == nil || len(lo.Filters) == 0 { - lo.Filters = make([]sqltypes.OrFilter, 1) - lo.Filters[0].Filters = make([]sqltypes.Filter, len(unboundSortLabels)) - i := 0 - for labelName := range unboundSortLabels { - lo.Filters[0].Filters[i] = sqltypes.Filter{ - Field: []string{"metadata", "labels", labelName}, - Op: sqltypes.Exists, - } - i++ - } - return - } - // The gotcha is we have to bind the labels for each set of orFilters, so copy them each time - for i, orFilters := range lo.Filters { - copyUnboundSortLabels := make(map[string]bool, len(unboundSortLabels)) - for k, v := range unboundSortLabels { - copyUnboundSortLabels[k] = v - } - for _, filter := range orFilters.Filters { - if isLabelFilter(&filter) { - copyUnboundSortLabels[filter.Field[2]] = false - } - } - // Now for any labels that are still true, add another where clause - for labelName, needsBinding := range copyUnboundSortLabels { - if needsBinding { - // `orFilters` is a copy of lo.Filters[i], so reference the original. - lo.Filters[i].Filters = append(lo.Filters[i].Filters, sqltypes.Filter{ - Field: []string{"metadata", "labels", labelName}, - Op: sqltypes.Exists, - }) - } - } - } -} - -// Possible ops from the k8s parser: -// KEY = and == (same) VALUE -// KEY != VALUE -// KEY exists [] # ,KEY, => this filter -// KEY ! [] # ,!KEY, => assert KEY doesn't exist -// KEY in VALUES -// KEY notin VALUES - -func (l *ListOptionIndexer) getFieldFilter(filter sqltypes.Filter) (string, []any, error) { - opString := "" - escapeString := "" - columnName := toColumnName(filter.Field) - if err := l.validateColumn(columnName); err != nil { - return "", nil, err - } - switch filter.Op { - case sqltypes.Eq: - if filter.Partial { - opString = "LIKE" - escapeString = escapeBackslashDirective - } else { - opString = "=" - } - clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) - return clause, []any{formatMatchTarget(filter)}, nil - case sqltypes.NotEq: - if filter.Partial { - opString = "NOT LIKE" - escapeString = escapeBackslashDirective - } else { - opString = "!=" - } - clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) - return clause, []any{formatMatchTarget(filter)}, nil - - case sqltypes.Lt, sqltypes.Gt: - sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`f."%s" %s ?`, columnName, sym) - return clause, []any{target}, nil - - case sqltypes.Exists, sqltypes.NotExists: - return "", nil, errors.New("NULL and NOT NULL tests aren't supported for non-label queries") - - case sqltypes.In: - fallthrough - case sqltypes.NotIn: - target := "()" - if len(filter.Matches) > 0 { - target = fmt.Sprintf("(?%s)", strings.Repeat(", ?", len(filter.Matches)-1)) - } - opString = "IN" - if filter.Op == sqltypes.NotIn { - opString = "NOT IN" - } - clause := fmt.Sprintf(`f."%s" %s %s`, columnName, opString, target) - matches := make([]any, len(filter.Matches)) - for i, match := range filter.Matches { - matches[i] = match - } - return clause, matches, nil - } - - return "", nil, fmt.Errorf("unrecognized operator: %s", opString) -} - -func (l *ListOptionIndexer) getLabelFilter(index int, filter sqltypes.Filter, dbName string) (string, []any, error) { - opString := "" - escapeString := "" - matchFmtToUse := strictMatchFmt - labelName := filter.Field[2] - switch filter.Op { - case sqltypes.Eq: - if filter.Partial { - opString = "LIKE" - escapeString = escapeBackslashDirective - matchFmtToUse = matchFmt - } else { - opString = "=" - } - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?%s`, index, index, opString, escapeString) - return clause, []any{labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil - - case sqltypes.NotEq: - if filter.Partial { - opString = "NOT LIKE" - escapeString = escapeBackslashDirective - matchFmtToUse = matchFmt - } else { - opString = "!=" - } - subFilter := sqltypes.Filter{ - Field: filter.Field, - Op: sqltypes.NotExists, - } - existenceClause, subParams, err := l.getLabelFilter(index, subFilter, dbName) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value %s ?%s)`, existenceClause, index, index, opString, escapeString) - params := append(subParams, labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) - return clause, params, nil - - case sqltypes.Lt, sqltypes.Gt: - sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?`, index, index, sym) - return clause, []any{labelName, target}, nil - - case sqltypes.Exists: - clause := fmt.Sprintf(`lt%d.label = ?`, index) - return clause, []any{labelName}, nil - - case sqltypes.NotExists: - clause := fmt.Sprintf(`o.key NOT IN (SELECT o1.key FROM "%s" o1 - JOIN "%s_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "%s_labels" lt%di1 ON o1.key = lt%di1.key - WHERE lt%di1.label = ?)`, dbName, dbName, dbName, index, index, index) - return clause, []any{labelName}, nil - - case sqltypes.In: - target := "(?" - if len(filter.Matches) > 0 { - target += strings.Repeat(", ?", len(filter.Matches)-1) - } - target += ")" - clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value IN %s`, index, index, target) - matches := make([]any, len(filter.Matches)+1) - matches[0] = labelName - for i, match := range filter.Matches { - matches[i+1] = match - } - return clause, matches, nil - - case sqltypes.NotIn: - target := "(?" - if len(filter.Matches) > 0 { - target += strings.Repeat(", ?", len(filter.Matches)-1) - } - target += ")" - subFilter := sqltypes.Filter{ - Field: filter.Field, - Op: sqltypes.NotExists, - } - existenceClause, subParams, err := l.getLabelFilter(index, subFilter, dbName) - if err != nil { - return "", nil, err - } - clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value NOT IN %s)`, existenceClause, index, index, target) - matches := append(subParams, labelName) - for _, match := range filter.Matches { - matches = append(matches, match) - } - return clause, matches, nil - } - return "", nil, fmt.Errorf("unrecognized operator: %s", opString) -} - -func prepareComparisonParameters(op sqltypes.Op, target string) (string, float64, error) { - num, err := strconv.ParseFloat(target, 32) - if err != nil { - return "", 0, err - } - switch op { - case sqltypes.Lt: - return "<", num, nil - case sqltypes.Gt: - return ">", num, nil - } - return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op) -} - -func formatMatchTarget(filter sqltypes.Filter) string { - format := strictMatchFmt - if filter.Partial { - format = matchFmt - } - return formatMatchTargetWithFormatter(filter.Matches[0], format) -} - -func formatMatchTargetWithFormatter(match string, format string) string { - // To allow matches on the backslash itself, the character needs to be replaced first. - // Otherwise, it will undo the following replacements. - match = strings.ReplaceAll(match, `\`, `\\`) - match = strings.ReplaceAll(match, `_`, `\_`) - match = strings.ReplaceAll(match, `%`, `\%`) - return fmt.Sprintf(format, match) -} - -// There are two kinds of string arrays to turn into a string, based on the last value in the array -// simple: ["a", "b", "conformsToIdentifier"] => "a.b.conformsToIdentifier" -// complex: ["a", "b", "foo.io/stuff"] => "a.b[foo.io/stuff]" - -func smartJoin(s []string) string { - if len(s) == 0 { - return "" - } - if len(s) == 1 { - return s[0] - } - lastBit := s[len(s)-1] - simpleName := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) - if simpleName.MatchString(lastBit) { - return strings.Join(s, ".") - } - return fmt.Sprintf("%s[%s]", strings.Join(s[0:len(s)-1], "."), lastBit) -} - -// toColumnName returns the column name corresponding to a field expressed as string slice -func toColumnName(s []string) string { - return db.Sanitize(smartJoin(s)) + return subfields } // getField extracts the value of a field expressed as a string path from an unstructured object @@ -920,31 +391,9 @@ func getField(a any, field string) (any, error) { return obj, nil } -func extractSubFields(fields string) []string { - subfields := make([]string, 0) - for _, subField := range subfieldRegex.FindAllString(fields, -1) { - subfields = append(subfields, strings.TrimSuffix(subField, ".")) - } - return subfields -} - -func isLabelFilter(f *sqltypes.Filter) bool { - return len(f.Field) >= 2 && f.Field[0] == "metadata" && f.Field[1] == "labels" -} - -func hasLabelFilter(filters []sqltypes.OrFilter) bool { - for _, outerFilter := range filters { - for _, filter := range outerFilter.Filters { - if isLabelFilter(&filter) { - return true - } - } - } - return false -} - -func isLabelsFieldList(fields []string) bool { - return len(fields) == 3 && fields[0] == "metadata" && fields[1] == "labels" +// toColumnName returns the column name corresponding to a field expressed as string slice +func toColumnName(s []string) string { + return db.Sanitize(smartJoin(s)) } // toUnstructuredList turns a slice of unstructured objects into an unstructured.UnstructuredList diff --git a/pkg/sqlcache/informer/listoption_indexer_test.go b/pkg/sqlcache/informer/listoption_indexer_test.go index 4248edf1..39f226ad 100644 --- a/pkg/sqlcache/informer/listoption_indexer_test.go +++ b/pkg/sqlcache/informer/listoption_indexer_test.go @@ -9,20 +9,12 @@ package informer import ( "context" "database/sql" - "errors" "fmt" - "github.com/rancher/steve/pkg/sqlcache/sqltypes" - "reflect" - "strings" "testing" "github.com/rancher/steve/pkg/sqlcache/db" - "github.com/rancher/steve/pkg/sqlcache/partition" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" ) func TestNewListOptionIndexer(t *testing.T) { @@ -252,1541 +244,3 @@ func TestNewListOptionIndexer(t *testing.T) { t.Run(test.description, func(t *testing.T) { test.test(t) }) } } - -func TestListByOptions(t *testing.T) { - type testCase struct { - description string - listOptions sqltypes.ListOptions - partitions []partition.Partition - ns string - expectedCountStmt string - expectedCountStmtArgs []any - expectedStmt string - expectedStmtArgs []any - extraIndexedFields []string - expectedList *unstructured.UnstructuredList - returnList []any - expectedContToken string - expectedErr error - } - - testObject := testStoreObject{Id: "something", Val: "a"} - unstrTestObjectMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testObject) - assert.Nil(t, err) - - var tests []testCase - tests = append(tests, testCase{ - description: "ListByOptions() with no errors returned, should not return an error", - listOptions: sqltypes.ListOptions{}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions() with an empty filter, should not return an error", - listOptions: sqltypes.ListOptions{ - Filters: []sqltypes.OrFilter{{[]sqltypes.Filter{}}}, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ChunkSize: 2}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ?`, - expectedStmtArgs: []interface{}{2}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Resume set should set offset in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Resume: "4"}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - OFFSET ?`, - expectedStmtArgs: []interface{}{4}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter with Op set top NotEq should select where that filter is not true in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: sqltypes.NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" NOT LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with 1 filter with Partial set to true should select where that partial match on that filter's value is true in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.somefield" LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with 1 OrFilter set with multiple filters should select where any of those filters are true in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"someothervalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"somethirdvalue"}, - Op: sqltypes.NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - ((f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"%somevalue%", "%someothervalue%", "%somethirdvalue%"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with multiple OrFilters set should select where all OrFilters contain one filter that is true in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"value1"}, - Op: sqltypes.Eq, - Partial: false, - }, - { - Field: []string{"status", "someotherfield"}, - Matches: []string{"value2"}, - Op: sqltypes.NotEq, - Partial: false, - }, - }, - }, - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"value3"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test4", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - ((f."metadata.somefield" = ?) OR (f."status.someotherfield" != ?)) AND - (f."metadata.somefield" = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"value1", "value2", "value3", "test4"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with labels filter should select the label in the prepared sql.Stmt", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "guard.cattle.io"}, - Matches: []string{"lodgepole"}, - Op: sqltypes.Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test41", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"guard.cattle.io", "%lodgepole%", "test41"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with two labels filters should use a self-join", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "cows"}, - Matches: []string{"milk"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "horses"}, - Matches: []string{"saddles"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test42", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (lt2.label = ? AND lt2.value = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"cows", "milk", "horses", "saddles", "test42"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with a mix of one label and one non-label query can still self-join", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "cows"}, - Matches: []string{"butter"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "somefield"}, - Matches: []string{"wheat"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test43", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (f."metadata.somefield" = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"cows", "butter", "wheat", "test43"}, - returnList: []any{}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with only one Sort.Field set should sort on that field only, in ascending order in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "somefield"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test5", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.somefield" ASC`, - expectedStmtArgs: []any{"test5"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "sort one field descending", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "somefield"}, - Order: sqltypes.DESC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test5a", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY f."metadata.somefield" DESC`, - expectedStmtArgs: []any{"test5a"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "sort one unbound label descending", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "labels", "flip"}, - Order: sqltypes.DESC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "test5a", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (f."metadata.namespace" = ?) AND - (FALSE) - ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) DESC NULLS FIRST`, - expectedStmtArgs: []any{"flip", "test5a", "flip"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions sorting on two complex fields should sort on the first field in ascending order first and then sort on the second labels field in ascending order in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "fields", "3"}, - Order: sqltypes.ASC, - }, - { - Fields: []string{"metadata", "labels", "stub.io/candy"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - extraIndexedFields: []string{"metadata.fields[3]", "metadata.labels[stub.io/candy]"}, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY f."metadata.fields[3]" ASC, (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, - expectedStmtArgs: []any{"stub.io/candy", "stub.io/candy"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions sorting on two fields should sort on the first field in ascending order first and then sort on the second field in ascending order in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "somefield"}, - Order: sqltypes.ASC, - }, - { - Fields: []string{"status", "someotherfield"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.somefield" ASC, f."status.someotherfield" ASC`, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions sorting on two fields should sort on the first field in descending order first and then sort on the second field in ascending order in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "somefield"}, - Order: sqltypes.DESC, - }, - { - Fields: []string{"status", "someotherfield"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.somefield" DESC, f."status.someotherfield" ASC`, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.PageSize set should set limit to PageSize in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - Pagination: sqltypes.Pagination{ - PageSize: 10, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ?`, - expectedStmtArgs: []any{10}, - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.Page and no PageSize set should not add anything to prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - Pagination: sqltypes.Pagination{ - Page: 2, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with Pagination.Page and PageSize set limit to PageSize and offset to PageSize * (Page - 1) in prepared sql.Stmt", - listOptions: sqltypes.ListOptions{ - Pagination: sqltypes.Pagination{ - PageSize: 10, - Page: 2, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE) - ORDER BY f."metadata.name" ASC - LIMIT ? - OFFSET ?`, - expectedStmtArgs: []any{10, 10}, - - expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (FALSE))`, - expectedCountStmtArgs: []any{}, - - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Namespace Partition should select only items where metadata.namespace is equal to Namespace and all other conditions are met in prepared sql.Stmt", - partitions: []partition.Partition{ - { - Namespace: "somens", - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.namespace" = ? AND FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somens"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a All Partition should select all items that meet all other conditions in prepared sql.Stmt", - partitions: []partition.Partition{ - { - All: true, - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Passthrough Partition should select all items that meet all other conditions prepared sql.Stmt", - partitions: []partition.Partition{ - { - Passthrough: true, - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - ORDER BY f."metadata.name" ASC `, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "ListByOptions with a Names Partition should select only items where metadata.name equals an items in Names and all other conditions are met in prepared sql.Stmt", - partitions: []partition.Partition{ - { - Names: sets.New[string]("someid", "someotherid"), - }, - }, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.name" IN (?, ?)) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"someid", "someotherid"}, - returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, - expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, - expectedContToken: "", - expectedErr: nil, - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - txClient := NewMockTXClient(gomock.NewController(t)) - store := NewMockStore(gomock.NewController(t)) - stmts := NewMockStmt(gomock.NewController(t)) - i := &Indexer{ - Store: store, - } - lii := &ListOptionIndexer{ - Indexer: i, - indexedFields: []string{"metadata.somefield", "status.someotherfield"}, - } - if len(test.extraIndexedFields) > 0 { - lii.indexedFields = append(lii.indexedFields, test.extraIndexedFields...) - } - queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") - if test.expectedErr != nil { - assert.Equal(t, test.expectedErr, err) - return - } - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, queryInfo.query) - if test.expectedStmtArgs == nil { - test.expectedStmtArgs = []any{} - } - assert.Equal(t, test.expectedStmtArgs, queryInfo.params) - assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) - assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) - - stmt := &sql.Stmt{} - rows := &sql.Rows{} - objType := reflect.TypeOf(testObject) - txClient.EXPECT().Stmt(gomock.Any()).Return(stmts).AnyTimes() - store.EXPECT().Prepare(test.expectedStmt).Do(func(a ...any) { - fmt.Println(a) - }).Return(stmt) - if args := test.expectedStmtArgs; args != nil { - stmts.EXPECT().QueryContext(gomock.Any(), gomock.Any()).Return(rows, nil).AnyTimes() - } else if strings.Contains(test.expectedStmt, "LIMIT") { - stmts.EXPECT().QueryContext(gomock.Any(), args...).Return(rows, nil) - txClient.EXPECT().Stmt(gomock.Any()).Return(stmts) - stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) - } else { - stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) - } - store.EXPECT().GetType().Return(objType) - store.EXPECT().GetShouldEncrypt().Return(false) - store.EXPECT().ReadObjects(rows, objType, false).Return(test.returnList, nil) - store.EXPECT().CloseStmt(stmt).Return(nil) - - store.EXPECT().WithTransaction(gomock.Any(), false, gomock.Any()).Return(nil).Do( - func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { - err := f(txClient) - if test.expectedErr == nil { - assert.Nil(t, err) - } else { - assert.Equal(t, test.expectedErr, err) - } - }) - - if test.expectedCountStmt != "" { - store.EXPECT().Prepare(test.expectedCountStmt).Return(stmt) - store.EXPECT().ReadInt(rows).Return(len(test.expectedList.Items), nil) - store.EXPECT().CloseStmt(stmt).Return(nil) - } - list, total, contToken, err := lii.executeQuery(context.Background(), queryInfo) - if test.expectedErr == nil { - assert.Nil(t, err) - } else { - assert.Equal(t, test.expectedErr, err) - } - assert.Equal(t, test.expectedList, list) - assert.Equal(t, len(test.expectedList.Items), total) - assert.Equal(t, test.expectedContToken, contToken) - }) - } -} - -func TestConstructQuery(t *testing.T) { - type testCase struct { - description string - listOptions sqltypes.ListOptions - partitions []partition.Partition - ns string - expectedCountStmt string - expectedCountStmtArgs []any - expectedStmt string - expectedStmtArgs []any - expectedErr error - } - - var tests []testCase - tests = append(tests, testCase{ - description: "TestConstructQuery: handles IN statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"somevalue"}, - Op: sqltypes.In, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.queryField1" IN (?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOT-IN statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"somevalue"}, - Op: sqltypes.NotIn, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - WHERE - (f."metadata.queryField1" NOT IN (?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles EXISTS statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "queryField1"}, - Op: sqltypes.Exists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOT-EXISTS statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "queryField1"}, - Op: sqltypes.NotExists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelEqualFull"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEqualFull", "somevalue"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements, match partial", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelEqualPartial"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles != statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelNotEqualFull"}, - Matches: []string{"somevalue"}, - Op: sqltypes.NotEq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNotEqualFull", "labelNotEqualFull", "somevalue"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles != statements for label statements, match partial", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelNotEqualPartial"}, - Matches: []string{"somevalue"}, - Op: sqltypes.NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNotEqualPartial", "labelNotEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles multiple != statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "notEqual1"}, - Matches: []string{"value1"}, - Op: sqltypes.NotEq, - Partial: false, - }, - }, - }, - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "notEqual2"}, - Matches: []string{"value2"}, - Op: sqltypes.NotEq, - Partial: false, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt2i1 ON o1.key = lt2i1.key - WHERE lt2i1.label = ?)) OR (lt2.label = ? AND lt2.value != ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"notEqual1", "notEqual1", "value1", "notEqual2", "notEqual2", "value2"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles IN statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelIN"}, - Matches: []string{"somevalue1", "someValue2"}, - Op: sqltypes.In, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value IN (?, ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelIN", "somevalue1", "someValue2"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOTIN statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelNOTIN"}, - Matches: []string{"somevalue1", "someValue2"}, - Op: sqltypes.NotIn, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT IN (?, ?))) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNOTIN", "labelNOTIN", "somevalue1", "someValue2"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles EXISTS statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelEXISTS"}, - Matches: []string{}, - Op: sqltypes.Exists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelEXISTS"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles NOTEXISTS statements for label statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelNOTEXISTS"}, - Matches: []string{}, - Op: sqltypes.NotExists, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (o.key NOT IN (SELECT o1.key FROM "something" o1 - JOIN "something_fields" f1 ON o1.key = f1.key - LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key - WHERE lt1i1.label = ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"labelNOTEXISTS"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles LessThan statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "numericThing"}, - Matches: []string{"5"}, - Op: sqltypes.Lt, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value < ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"numericThing", float64(5)}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "TestConstructQuery: handles GreaterThan statements", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "numericThing"}, - Matches: []string{"35"}, - Op: sqltypes.Gt, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value > ?) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"numericThing", float64(35)}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "multiple filters with a positive label test and a negative non-label test still outer-join", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "junta"}, - Matches: []string{"esther"}, - Op: sqltypes.Eq, - Partial: true, - }, - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"golgi"}, - Op: sqltypes.NotEq, - Partial: true, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" NOT LIKE ? ESCAPE '\')) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"junta", "%esther%", "%golgi%"}, - expectedErr: nil, - }) - tests = append(tests, testCase{ - description: "multiple filters and or-filters with a positive label test and a negative non-label test still outer-join and have correct AND/ORs", - listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "nectar"}, - Matches: []string{"stash"}, - Op: sqltypes.Eq, - Partial: true, - }, - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"landlady"}, - Op: sqltypes.NotEq, - Partial: false, - }, - }, - }, - { - Filters: []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "lawn"}, - Matches: []string{"reba", "coil"}, - Op: sqltypes.In, - }, - { - Field: []string{"metadata", "queryField1"}, - Op: sqltypes.Gt, - Matches: []string{"2"}, - }, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - WHERE - ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" != ?)) AND - ((lt2.label = ? AND lt2.value IN (?, ?)) OR (f."metadata.queryField1" > ?)) AND - (FALSE) - ORDER BY f."metadata.name" ASC `, - expectedStmtArgs: []any{"nectar", "%stash%", "landlady", "lawn", "reba", "coil", float64(2)}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: handles == statements for label statements, match partial, sort on metadata.queryField1", - listOptions: sqltypes.ListOptions{ - Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "labels", "labelEqualPartial"}, - Matches: []string{"somevalue"}, - Op: sqltypes.Eq, - Partial: true, - }, - }, - }, - }, - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "queryField1"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND - (FALSE) - ORDER BY f."metadata.queryField1" ASC`, - expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: sort on label statements with no query", - listOptions: sqltypes.ListOptions{ - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "labels", "this"}, - Order: sqltypes.ASC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key - WHERE - (lt1.label = ?) AND - (FALSE) - ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, - expectedStmtArgs: []any{"this", "this"}, - expectedErr: nil, - }) - - tests = append(tests, testCase{ - description: "TestConstructQuery: sort and query on both labels and non-labels without overlap", - listOptions: sqltypes.ListOptions{ - Filters: []sqltypes.OrFilter{ - { - []sqltypes.Filter{ - { - Field: []string{"metadata", "queryField1"}, - Matches: []string{"toys"}, - Op: sqltypes.Eq, - }, - { - Field: []string{"metadata", "labels", "jamb"}, - Matches: []string{"juice"}, - Op: sqltypes.Eq, - }, - }, - }, - }, - SortList: sqltypes.SortList{ - SortDirectives: []sqltypes.Sort{ - { - Fields: []string{"metadata", "labels", "this"}, - Order: sqltypes.ASC, - }, - { - Fields: []string{"status", "queryField2"}, - Order: sqltypes.DESC, - }, - }, - }, - }, - partitions: []partition.Partition{}, - ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o - JOIN "something_fields" f ON o.key = f.key - LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key - LEFT OUTER JOIN "something_labels" lt3 ON o.key = lt3.key - WHERE - ((f."metadata.queryField1" = ?) OR (lt2.label = ? AND lt2.value = ?) OR (lt3.label = ?)) AND - (FALSE) - ORDER BY (CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST, f."status.queryField2" DESC`, - expectedStmtArgs: []any{"toys", "jamb", "juice", "this", "this"}, - expectedErr: nil, - }) - - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - store := NewMockStore(gomock.NewController(t)) - i := &Indexer{ - Store: store, - } - lii := &ListOptionIndexer{ - Indexer: i, - indexedFields: []string{"metadata.queryField1", "status.queryField2"}, - } - queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") - if test.expectedErr != nil { - assert.Equal(t, test.expectedErr, err) - return - } - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, queryInfo.query) - assert.Equal(t, test.expectedStmtArgs, queryInfo.params) - assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) - assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) - }) - } -} - -func TestSmartJoin(t *testing.T) { - type testCase struct { - description string - fieldArray []string - expectedFieldName string - } - - var tests []testCase - tests = append(tests, testCase{ - description: "single-letter names should be dotted", - fieldArray: []string{"metadata", "labels", "a"}, - expectedFieldName: "metadata.labels.a", - }) - tests = append(tests, testCase{ - description: "underscore should be dotted", - fieldArray: []string{"metadata", "labels", "_"}, - expectedFieldName: "metadata.labels._", - }) - tests = append(tests, testCase{ - description: "simple names should be dotted", - fieldArray: []string{"metadata", "labels", "queryField2"}, - expectedFieldName: "metadata.labels.queryField2", - }) - tests = append(tests, testCase{ - description: "a numeric field should be bracketed", - fieldArray: []string{"metadata", "fields", "43"}, - expectedFieldName: "metadata.fields[43]", - }) - tests = append(tests, testCase{ - description: "a field starting with a number should be bracketed", - fieldArray: []string{"metadata", "fields", "46days"}, - expectedFieldName: "metadata.fields[46days]", - }) - tests = append(tests, testCase{ - description: "compound names should be bracketed", - fieldArray: []string{"metadata", "labels", "rancher.cattle.io/moo"}, - expectedFieldName: "metadata.labels[rancher.cattle.io/moo]", - }) - tests = append(tests, testCase{ - description: "space-separated names should be bracketed", - fieldArray: []string{"metadata", "labels", "space here"}, - expectedFieldName: "metadata.labels[space here]", - }) - tests = append(tests, testCase{ - description: "already-bracketed terms cause double-bracketing and should never be used", - fieldArray: []string{"metadata", "labels[k8s.io/deepcode]"}, - expectedFieldName: "metadata[labels[k8s.io/deepcode]]", - }) - tests = append(tests, testCase{ - description: "an empty array should be an empty string", - fieldArray: []string{}, - expectedFieldName: "", - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - gotFieldName := smartJoin(test.fieldArray) - assert.Equal(t, test.expectedFieldName, gotFieldName) - }) - } -} - -func TestBuildSortLabelsClause(t *testing.T) { - type testCase struct { - description string - labelName string - joinTableIndexByLabelName map[string]int - direction bool - expectedStmt string - expectedParam string - expectedErr string - } - - var tests []testCase - tests = append(tests, testCase{ - description: "TestBuildSortClause: empty index list errors", - labelName: "emptyListError", - expectedErr: `internal error: no join-table index given for labelName "emptyListError"`, - }) - tests = append(tests, testCase{ - description: "TestBuildSortClause: hit ascending", - labelName: "testBSL1", - joinTableIndexByLabelName: map[string]int{"testBSL1": 3}, - direction: true, - expectedStmt: `(CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST`, - expectedParam: "testBSL1", - }) - tests = append(tests, testCase{ - description: "TestBuildSortClause: hit descending", - labelName: "testBSL2", - joinTableIndexByLabelName: map[string]int{"testBSL2": 4}, - direction: false, - expectedStmt: `(CASE lt4.label WHEN ? THEN lt4.value ELSE NULL END) DESC NULLS FIRST`, - expectedParam: "testBSL2", - }) - t.Parallel() - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - stmt, param, err := buildSortLabelsClause(test.labelName, test.joinTableIndexByLabelName, test.direction) - if test.expectedErr != "" { - assert.Equal(t, test.expectedErr, err.Error()) - } else { - assert.Nil(t, err) - assert.Equal(t, test.expectedStmt, stmt) - assert.Equal(t, test.expectedParam, param) - } - }) - } -} diff --git a/pkg/sqlcache/informer/query_generator.go b/pkg/sqlcache/informer/query_generator.go new file mode 100644 index 00000000..22dd77ab --- /dev/null +++ b/pkg/sqlcache/informer/query_generator.go @@ -0,0 +1,1102 @@ +package informer + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" +) + +var ( + badTableNameChars = regexp.MustCompile(`[^-a-zA-Z0-9._]+`) + nonIdentifierChars = regexp.MustCompile(`[^a-zA-Z0-9_]+`) +) + +func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) { + indirectSortDirective, err := checkForIndirectSortDirective(lo) + if err != nil { + return nil, err + } + joinTableIndexByLabelName := make(map[string]int) + if indirectSortDirective != nil && isLabelsFieldList(indirectSortDirective.Fields) { + return l.constructIndirectSortQuery(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + } + ensureSortLabelsAreSelected(lo) + return l.finishConstructQuery(lo, partitions, namespace, dbName, joinTableIndexByLabelName) +} + +/** constructIndirectSortQuery - process indirect-sorting + * Here we create two queries: + * one that has an existence test for the sorter, + * and one with a non-existence test, so each of these is AND-ed with the other WHERE tests (filters). + * Then do a `UNION ALL` on the two different queries. + * The unobvious part: have the original options-list do only the one indirect sort. + * Have the copy process any other sort options. + * + * Two limitations: + * 1. Only at most one indirect sort per query + * 2. The indirect sort will go before the other ones (todo: fix this) + */ + +func (l *ListOptionIndexer) constructIndirectSortQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) (*QueryInfo, error) { + var loNoLabel sqltypes.ListOptions + // We want to make sure that the want-sort-label options test for the label's existence, + // but we want the non-label to have a not-exists test on it. So first ensure it exists, + // then ensure a non-exists test exists on the non-label filter. + // The other thing is we put all the non-indirect sort directives on the copy of the list options + var indirectSortDirective sqltypes.Sort + newSortList1 := make([]sqltypes.Sort, 1) + newSortList2 := make([]sqltypes.Sort, 0, len(lo.SortList.SortDirectives)-1) + indirectSortPosition := -1 + for i, sd := range lo.SortList.SortDirectives { + if sd.IsIndirect { + indirectSortDirective = lo.SortList.SortDirectives[i] + newSortList1[0] = indirectSortDirective + indirectSortPosition = i + } else { + newSortList2 = append(newSortList2, sd) + } + } + if indirectSortPosition == -1 { + return nil, fmt.Errorf("expected an indirect sort directive, didn't find one") + } + if len(indirectSortDirective.IndirectFields) != 4 { + return nil, fmt.Errorf("expected indirect sort directive to have 4 indirect fields, got %d", len(indirectSortDirective.IndirectFields)) + } + bytes, err := json.Marshal(*lo) + if err != nil { + return nil, fmt.Errorf("can't json-encode list options: %w", err) + } + err = json.Unmarshal(bytes, &loNoLabel) + if err != nil { + return nil, fmt.Errorf("can't json-decode list options: %w", err) + } + applyIndirectLabelTests(lo, &loNoLabel, &indirectSortDirective) + lo.SortList.SortDirectives = newSortList1 + loNoLabel.SortList.SortDirectives = newSortList2 + joinParts1, whereClauses1, params1, needsDistinctModifier1, _, _, _, err1 := l.getQueryParts(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + if err1 != nil { + return nil, err1 + } + // Now add clauses for the indirectSortDirective + joinParts2, whereClauses2, params2, needsDistinctModifier2, orderByClauses2, orderByParams2, _, err2 := l.getQueryParts(&loNoLabel, partitions, namespace, dbName, joinTableIndexByLabelName) + if err2 != nil { + return nil, err2 + } + addFalseTest := false + if whereClauses1[len(whereClauses1)-1] == "FALSE" { + whereClauses1 = whereClauses1[:len(whereClauses1)-1] + addFalseTest = true + } + if whereClauses2[len(whereClauses2)-1] == "FALSE" { + whereClauses2 = whereClauses2[:len(whereClauses2)-1] + addFalseTest = true + } + distinctModifier := "" + if needsDistinctModifier1 || needsDistinctModifier2 { + distinctModifier = " DISTINCT" + } + + externalTableName := getExternalTableName(&indirectSortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + return nil, fmt.Errorf("internal error: unable to find an entry for external table %s", externalTableName) + } + sortParts, importWithParts, importAsNullParts := processOrderByFields(&indirectSortDirective, extIndex, orderByClauses2) + selectLine := fmt.Sprintf("SELECT%s o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid", distinctModifier) + indent1 := " " + indent2 := indent1 + indent1 + indent3 := indent2 + indent1 + where1 := joinWhereClauses(whereClauses1, indent2, indent3, "AND") + where2 := joinWhereClauses(whereClauses2, indent2, indent3, "AND") + + parts := []string{ + "SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM (", + fmt.Sprintf(`%s%s, %s FROM %s`, indent1, selectLine, strings.Join(importWithParts, ", "), strings.Join(joinParts1, "\n"+indent2)), + where1, + "UNION ALL", + fmt.Sprintf(`%s%s, %s FROM %s`, indent1, selectLine, strings.Join(importAsNullParts, ", "), strings.Join(joinParts2, "\n"+indent2)), + where2, + ")", + } + if addFalseTest { + parts = append(parts, "WHERE FALSE") + } + params := make([]any, 0, len(params1)+len(params2)+len(orderByParams2)) + params = append(params, params1...) + params = append(params, params2...) + fullQuery := strings.Join(parts, "\n") + countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", fullQuery) + countParams := params[:] + params = append(params, orderByParams2...) + + fixedSortParts := putIndirectSortInPosition(sortParts, indirectSortPosition) + fullQuery += fmt.Sprintf("\n%sORDER BY %s", indent1, strings.Join(fixedSortParts, ", ")) + queryInfo := &QueryInfo{ + query: fullQuery, + params: params, + countQuery: countQuery, + countParams: countParams, + } + return queryInfo, nil +} + +func putIndirectSortInPosition(sortParts []string, indirectSortPosition int) []string { + fixedSortParts := make([]string, 0, len(sortParts)) + indirectSortPart := sortParts[0] + sortParts = sortParts[1:] + fixedSortParts = append(fixedSortParts, sortParts[0:indirectSortPosition]...) + fixedSortParts = append(fixedSortParts, indirectSortPart) + fixedSortParts = append(fixedSortParts, sortParts[indirectSortPosition:len(sortParts)]...) + return fixedSortParts +} + +func (l *ListOptionIndexer) finishConstructQuery(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) (*QueryInfo, error) { + + joinParts, whereClauses, params, needsDistinctModifier, orderByClauses, orderByParams, sortSelectField, err := l.getQueryParts(lo, partitions, namespace, dbName, joinTableIndexByLabelName) + if err != nil { + return nil, err + } + distinctModifier := "" + if needsDistinctModifier { + distinctModifier = " DISTINCT" + } + queryInfo := &QueryInfo{} + + if len(sortSelectField) > 0 { + if sortSelectField[0] != ' ' { + sortSelectField = " " + sortSelectField + } + } + query := fmt.Sprintf(`SELECT%s o.object, o.objectnonce, o.dekid%s FROM `, distinctModifier, sortSelectField) + query += strings.Join(joinParts, "\n ") + + if len(whereClauses) > 0 { + indent := " " + separator := fmt.Sprintf(") AND\n%s(", indent) + query += fmt.Sprintf("\n WHERE\n%s(%s)", indent, strings.Join(whereClauses, separator)) + } + + // before proceeding, save a copy of the query and params without LIMIT/OFFSET/ORDER info + // for COUNTing all results later + countQuery := fmt.Sprintf("SELECT COUNT(*) FROM (%s)", query) + countParams := params[:] + if len(orderByClauses) > 0 { + query += "\n ORDER BY " + query += strings.Join(orderByClauses, ", ") + params = append(params, orderByParams...) + } + + // 4- sqltypes.Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume) + + limitClause := "" + // take the smallest limit between lo.Pagination and lo.ChunkSize + limit := lo.Pagination.PageSize + if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) { + limit = lo.ChunkSize + } + if limit > 0 { + limitClause = "\n LIMIT ?" + params = append(params, limit) + } + + // OFFSET clause (from lo.Pagination and/or lo.Resume) + offsetClause := "" + offset := 0 + if lo.Resume != "" { + offsetInt, err := strconv.Atoi(lo.Resume) + if err != nil { + return queryInfo, err + } + offset = offsetInt + } + if lo.Pagination.Page >= 1 { + offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1) + } + if offset > 0 { + offsetClause = "\n OFFSET ?" + params = append(params, offset) + } + if limit > 0 || offset > 0 { + query += limitClause + query += offsetClause + queryInfo.countQuery = countQuery + queryInfo.countParams = countParams + queryInfo.limit = limit + queryInfo.offset = offset + } + // Otherwise leave these as default values and the executor won't do pagination work + + queryInfo.query = query + queryInfo.params = params + + return queryInfo, nil +} + +// Other ListOptionIndexer methods for generating SQL in alphabetical order: + +// buildORClause creates an SQLite compatible query that ORs conditions built from passed filters +func (l *ListOptionIndexer) buildORClauseFromFilters(orFilters sqltypes.OrFilter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, bool, error) { + params := make([]any, 0) + whereClauses := make([]string, 0, len(orFilters.Filters)) + joinClauses := make([]string, 0) + needDistinct := false + + for _, filter := range orFilters.Filters { + if isLabelFilter(&filter) { + fullName := fmt.Sprintf("%s:%s", dbName, filter.Field[2]) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + labelIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[fullName] = labelIndex + } + _, ok = joinedTables[fullName] + if !ok { + joinedTables[fullName] = true + joinClauses = append(joinClauses, fmt.Sprintf(`LEFT OUTER JOIN "%s_labels" lt%d ON o.key = lt%d.key`, dbName, labelIndex, labelIndex)) + } + needDistinct = true + labelFunc := l.getLabelFilter + if isIndirectFilter(&filter) { + labelFunc = l.getIndirectLabelFilter + } + newWhereClause, newJoins, newParams, err := labelFunc(filter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, needDistinct, err + } + joinClauses = append(joinClauses, newJoins...) + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } else if isIndirectFilter(&filter) { + newWhereClause, newJoins, newParams, err := l.getIndirectNonLabelFilter(filter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, needDistinct, err + } + joinClauses = append(joinClauses, newJoins...) + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } else { + newWhereClause, newParams, err := l.getFieldFilter(filter) + if err != nil { + return "", nil, nil, needDistinct, err + } + if newWhereClause != "" { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + } + } + finalWhereClause := "" + switch len(whereClauses) { + case 0: + finalWhereClause = "" // no change + case 1: + finalWhereClause = whereClauses[0] + default: + finalWhereClause = fmt.Sprintf("(%s)", strings.Join(whereClauses, ") OR (")) + } + return finalWhereClause, joinClauses, params, needDistinct, nil +} + +// ensureSortLabelsAreSelected - if the user tries to sort on a particular label without mentioning it in a query, +// and it's not an indirect sort directive, we need to ensure the label is added. +func ensureSortLabelsAreSelected(lo *sqltypes.ListOptions) { + if len(lo.SortList.SortDirectives) == 0 { + return + } + unboundSortLabels := make(map[string]bool) + for _, sortDirective := range lo.SortList.SortDirectives { + fields := sortDirective.Fields + if isLabelsFieldList(fields) { + unboundSortLabels[fields[2]] = true + } + } + if len(unboundSortLabels) == 0 { + return + } + // If we have sort directives but no filters, add an exists-filter for each label. + if lo.Filters == nil || len(lo.Filters) == 0 { + lo.Filters = make([]sqltypes.OrFilter, 1) + lo.Filters[0].Filters = make([]sqltypes.Filter, len(unboundSortLabels)) + i := 0 + for labelName := range unboundSortLabels { + lo.Filters[0].Filters[i] = sqltypes.Filter{ + Field: []string{"metadata", "labels", labelName}, + Op: sqltypes.Exists, + } + i++ + } + return + } + // Find any labels that are already mentioned in an existing filter + // The gotcha is we have to bind the labels for each set of orFilters, so copy them each time + for i, orFilters := range lo.Filters { + copyUnboundSortLabels := make(map[string]bool, len(unboundSortLabels)) + for k, v := range unboundSortLabels { + copyUnboundSortLabels[k] = v + } + for _, filter := range orFilters.Filters { + if isLabelFilter(&filter) { + copyUnboundSortLabels[filter.Field[2]] = false + } + } + // Now for any labels that are still true, add another where clause + for labelName, needsBinding := range copyUnboundSortLabels { + if needsBinding { + // `orFilters` is a copy of lo.Filters[i], so reference the original. + lo.Filters[i].Filters = append(lo.Filters[i].Filters, sqltypes.Filter{ + Field: []string{"metadata", "labels", labelName}, + Op: sqltypes.Exists, + }) + } + } + } +} + +// Possible ops from the k8s parser: +// KEY = and == (same) VALUE +// KEY != VALUE +// KEY exists [] # ,KEY, => this filter +// KEY ! [] # ,!KEY, => assert KEY doesn't exist +// KEY in VALUES +// KEY notin VALUES + +func (l *ListOptionIndexer) getFieldFilter(filter sqltypes.Filter) (string, []any, error) { + opString := "" + escapeString := "" + columnName := toColumnName(filter.Field) + if err := l.validateColumn(columnName); err != nil { + return "", nil, err + } + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + } else { + opString = "=" + } + clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) + return clause, []any{formatMatchTarget(filter)}, nil + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + } else { + opString = "!=" + } + clause := fmt.Sprintf(`f."%s" %s ?%s`, columnName, opString, escapeString) + return clause, []any{formatMatchTarget(filter)}, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, err + } + clause := fmt.Sprintf(`f."%s" %s ?`, columnName, sym) + return clause, []any{target}, nil + + case sqltypes.Exists, sqltypes.NotExists: + return "", nil, errors.New("NULL and NOT NULL tests aren't supported for non-label queries") + + case sqltypes.In: + fallthrough + case sqltypes.NotIn: + target := "()" + if len(filter.Matches) > 0 { + target = fmt.Sprintf("(?%s)", strings.Repeat(", ?", len(filter.Matches)-1)) + } + opString = "IN" + if filter.Op == sqltypes.NotIn { + opString = "NOT IN" + } + clause := fmt.Sprintf(`f."%s" %s %s`, columnName, opString, target) + matches := make([]any, len(filter.Matches)) + for i, match := range filter.Matches { + matches[i] = match + } + return clause, matches, nil + } + + return "", nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getIndirectLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + if len(filter.IndirectFields) != 4 { + s := "" + if len(filter.IndirectFields) > 0 { + s = strings.Join(filter.IndirectFields, " ") + } + return "", nil, nil, fmt.Errorf("expected exactly 4 indirect field parts, got %s (%d)", s, len(filter.IndirectFields)) + } + labelName := filter.Field[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", nil, nil, fmt.Errorf("internal error: can't find an entry for table %s", fullName) + } + + joinClauses := make([]string, 0) + + extDBName := fmt.Sprintf("%s_%s", filter.IndirectFields[0], filter.IndirectFields[1]) + extDBName = badTableNameChars.ReplaceAllString(extDBName, "_") + extDBName = strings.ReplaceAll(extDBName, "/", "_") + extIndex, ok := joinTableIndexByLabelName[extDBName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[extDBName] = extIndex + } + + selectorFieldName := filter.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + targetFieldName := filter.IndirectFields[3] + if badTableNameChars.MatchString(targetFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", targetFieldName) + } + extDBNameFields := fmt.Sprintf("%s_fields", extDBName) + _, ok = joinedTables[extDBNameFields] + if !ok { + joinedTables[extDBNameFields] = true + joinClauses = append(joinClauses, fmt.Sprintf(`JOIN "%s" ext%d ON lt%d.value = ext%d."%s"`, extDBNameFields, extIndex, labelIndex, extIndex, selectorFieldName)) + } + labelWhereSubClause := fmt.Sprintf("lt%d.label = ?", labelIndex) + targetFieldReference := fmt.Sprintf(`ext%d."%s"`, extIndex, targetFieldName) + var clause string + var op string + params := []any{labelName} + + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause = fmt.Sprintf(`%s AND %s %s ?%s`, labelWhereSubClause, targetFieldReference, opString, escapeString) + params = append(params, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + clause = fmt.Sprintf(`%s AND %s %s ?%s`, labelWhereSubClause, targetFieldReference, opString, escapeString) + params = append(params, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`%s AND %s %s ?`, labelWhereSubClause, targetFieldReference, sym) + params = append(params, target) + return clause, joinClauses, params, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`%s AND %s != NULL`, labelWhereSubClause, targetFieldReference) + return clause, joinClauses, params, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`%s AND %s == NULL`, labelWhereSubClause, targetFieldReference) + return clause, joinClauses, params, nil + + case sqltypes.In, sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + op = "IN" + if filter.Op == sqltypes.NotIn { + op = "NOT IN" + } + clause := fmt.Sprintf(`%s AND %s %s %s`, labelWhereSubClause, targetFieldReference, op, target) + for _, match := range filter.Matches { + params = append(params, match) + } + return clause, joinClauses, params, nil + + // See getLabelFilter for rest of operators + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getIndirectNonLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + if len(filter.IndirectFields) != 4 { + s := "" + if len(filter.IndirectFields) > 0 { + s = strings.Join(filter.IndirectFields, " ") + } + return "", nil, nil, fmt.Errorf("expected exactly 4 indirect field parts, got %s (%d)", s, len(filter.IndirectFields)) + } + columnName := toColumnName(filter.Field) + if err := l.validateColumn(columnName); err != nil { + return "", nil, nil, err + } + extDBName := fmt.Sprintf("%s_%s", filter.IndirectFields[0], filter.IndirectFields[1]) + extDBName = badTableNameChars.ReplaceAllString(extDBName, "_") + extIndex, ok := joinTableIndexByLabelName[extDBName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[extDBName] = extIndex + } + + selectorFieldName := filter.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := filter.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return "", nil, nil, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + extDBNameFields := fmt.Sprintf("%s_fields", extDBName) + _, ok = joinedTables[extDBNameFields] + joinClauses := make([]string, 0) + if !ok { + joinedTables[extDBNameFields] = true + joinClauses = append(joinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON f."%s" = ext%d."%s"`, extDBName, extIndex, columnName, extIndex, selectorFieldName)) + } + params := make([]any, 0) + + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause := fmt.Sprintf(`ext%d."%s" %s ?%s`, extIndex, externalFieldName, opString, escapeString) + return clause, joinClauses, []any{formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + clause := fmt.Sprintf(`ext%d."%s" %s ?%s`, extIndex, externalFieldName, opString, escapeString) + return clause, joinClauses, []any{formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`ext%d."%s" %s ?`, extIndex, externalFieldName, sym) + return clause, joinClauses, []any{target}, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`ext%d."%s" != NULL`, extIndex, externalFieldName) + return clause, joinClauses, []any{}, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`ext%d."%s" == NULL`, extIndex, externalFieldName) + return clause, joinClauses, []any{}, nil + + case sqltypes.In, sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + opString = "IN" + if filter.Op == sqltypes.NotIn { + opString = "NOT IN" + } + clause := fmt.Sprintf(`ext%d."%s" %s %s`, extIndex, externalFieldName, opString, target) + for _, match := range filter.Matches { + params = append(params, match) + } + return clause, joinClauses, params, nil + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getLabelFilter(filter sqltypes.Filter, dbName string, joinTableIndexByLabelName map[string]int, joinedTables map[string]bool) (string, []string, []any, error) { + opString := "" + escapeString := "" + matchFmtToUse := strictMatchFmt + labelName := filter.Field[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", nil, nil, fmt.Errorf("internal error: can't find an entry for table %s", fullName) + } + + joinClauses := make([]string, 0) + + switch filter.Op { + case sqltypes.Eq: + if filter.Partial { + opString = "LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "=" + } + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?%s`, labelIndex, labelIndex, opString, escapeString) + return clause, joinClauses, []any{labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)}, nil + + case sqltypes.NotEq: + if filter.Partial { + opString = "NOT LIKE" + escapeString = escapeBackslashDirective + matchFmtToUse = matchFmt + } else { + opString = "!=" + } + subFilter := sqltypes.Filter{ + Field: filter.Field, + Op: sqltypes.NotExists, + } + existenceClause, _, subParams, err := l.getLabelFilter(subFilter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value %s ?%s)`, existenceClause, labelIndex, labelIndex, opString, escapeString) + params := append(subParams, labelName, formatMatchTargetWithFormatter(filter.Matches[0], matchFmtToUse)) + return clause, joinClauses, params, nil + + case sqltypes.Lt, sqltypes.Gt: + sym, target, err := prepareComparisonParameters(filter.Op, filter.Matches[0]) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value %s ?`, labelIndex, labelIndex, sym) + return clause, joinClauses, []any{labelName, target}, nil + + case sqltypes.Exists: + clause := fmt.Sprintf(`lt%d.label = ?`, labelIndex) + return clause, joinClauses, []any{labelName}, nil + + case sqltypes.NotExists: + clause := fmt.Sprintf(`o.key NOT IN (SELECT o1.key FROM "%s" o1 + JOIN "%s_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "%s_labels" lt%di1 ON o1.key = lt%di1.key + WHERE lt%di1.label = ?)`, dbName, dbName, dbName, labelIndex, labelIndex, labelIndex) + return clause, joinClauses, []any{labelName}, nil + + case sqltypes.In: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + clause := fmt.Sprintf(`lt%d.label = ? AND lt%d.value IN %s`, labelIndex, labelIndex, target) + matches := make([]any, len(filter.Matches)+1) + matches[0] = labelName + for i, match := range filter.Matches { + matches[i+1] = match + } + return clause, joinClauses, matches, nil + + case sqltypes.NotIn: + target := "(?" + if len(filter.Matches) > 0 { + target += strings.Repeat(", ?", len(filter.Matches)-1) + } + target += ")" + subFilter := sqltypes.Filter{ + Field: filter.Field, + Op: sqltypes.NotExists, + } + existenceClause, _, subParams, err := l.getLabelFilter(subFilter, dbName, joinTableIndexByLabelName, joinedTables) + if err != nil { + return "", nil, nil, err + } + clause := fmt.Sprintf(`(%s) OR (lt%d.label = ? AND lt%d.value NOT IN %s)`, existenceClause, labelIndex, labelIndex, target) + matches := append(subParams, labelName) + for _, match := range filter.Matches { + matches = append(matches, match) + } + return clause, joinClauses, matches, nil + } + return "", nil, nil, fmt.Errorf("unrecognized operator: %s", opString) +} + +func (l *ListOptionIndexer) getQueryParts(lo *sqltypes.ListOptions, partitions []partition.Partition, namespace string, dbName string, joinTableIndexByLabelName map[string]int) ([]string, []string, []any, bool, []string, []any, string, error) { + joinParts := []string{fmt.Sprintf(`"%s" o`, dbName), fmt.Sprintf(`JOIN "%s_fields" f ON o.key = f.key`, dbName)} + whereClauses := make([]string, 0) + params := make([]any, 0) + needDistinctFinal := false + joinedTables := make(map[string]bool) + joinedTables[dbName] = true + joinedTables[fmt.Sprintf("%s_fields", dbName)] = true + // 1- Figure out what we'll be joining and testing + for _, orFilters := range lo.Filters { + newWhereClause, newJoinParts, newParams, needDistinct, err := l.buildORClauseFromFilters(orFilters, dbName, joinTableIndexByLabelName, joinedTables) + + if err != nil { + return joinParts, whereClauses, params, needDistinctFinal, nil, nil, "", err + } + joinParts = append(joinParts, newJoinParts...) + if len(newWhereClause) > 0 { + whereClauses = append(whereClauses, newWhereClause) + } + params = append(params, newParams...) + if needDistinct { + needDistinctFinal = true + } + } + + // WHERE clauses (from namespace) + if namespace != "" && namespace != "*" { + whereClauses = append(whereClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) + params = append(params, namespace) + } + + // WHERE clauses (from partitions and their corresponding parameters) + partitionClauses := make([]string, 0) + for _, thisPartition := range partitions { + if thisPartition.Passthrough { + // nothing to do, no extra filtering to apply by definition + } else { + singlePartitionClauses := make([]string, 0) + + // filter by namespace + if thisPartition.Namespace != "" && thisPartition.Namespace != "*" { + singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.namespace" = ?`)) + params = append(params, thisPartition.Namespace) + } + + // optionally filter by names + if !thisPartition.All { + names := thisPartition.Names + + if names.Len() == 0 { + // degenerate case, there will be no results + singlePartitionClauses = append(singlePartitionClauses, "FALSE") + } else { + singlePartitionClauses = append(singlePartitionClauses, fmt.Sprintf(`f."metadata.name" IN (?%s)`, strings.Repeat(", ?", thisPartition.Names.Len()-1))) + // sort for reproducibility + sortedNames := thisPartition.Names.UnsortedList() + sort.Strings(sortedNames) + for _, name := range sortedNames { + params = append(params, name) + } + } + } + + if len(singlePartitionClauses) > 0 { + partitionClauses = append(partitionClauses, strings.Join(singlePartitionClauses, " AND ")) + } + } + } + if len(partitions) == 0 { + // degenerate case, there will be no results + whereClauses = append(whereClauses, "FALSE") + } + if len(partitionClauses) == 1 { + whereClauses = append(whereClauses, partitionClauses[0]) + } + if len(partitionClauses) > 1 { + whereClauses = append(whereClauses, "(\n ("+strings.Join(partitionClauses, ") OR\n (")+")\n)") + } + sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err := l.getSortDirectives(lo, dbName, joinTableIndexByLabelName) + joinParts = append(joinParts, sortJoinClauses...) + whereClauses = append(whereClauses, sortWhereClauses...) + + return joinParts, whereClauses, params, needDistinctFinal, orderByClauses, orderByParams, sortSelectField, err +} + +func (l *ListOptionIndexer) getSortDirectives(lo *sqltypes.ListOptions, dbName string, joinTableIndexByLabelName map[string]int) (string, []string, []string, []string, []any, error) { + sortSelectField := "" + sortJoinClauses := make([]string, 0) + sortWhereClauses := make([]string, 0) + orderByClauses := make([]string, 0) + orderByParams := make([]any, 0) + if len(lo.SortList.SortDirectives) == 0 { + // make sure at least one default order is always picked + orderByClauses = append(orderByClauses, `f."metadata.name" ASC`) + if l.namespaced { + orderByClauses = append(orderByClauses, `f."metadata.namespace" ASC`) + } + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, nil + } + for _, sortDirective := range lo.SortList.SortDirectives { + fields := sortDirective.Fields + if isLabelsFieldList(fields) { + labelName := sortDirective.Fields[2] + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + if sortDirective.IsIndirect { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) + } + labelIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[fullName] = labelIndex + } + if sortDirective.IsIndirect { + //TODO: check the external table name. + externalTableName := getExternalTableName(&sortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[externalTableName] = extIndex + } + selectorFieldName := sortDirective.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := sortDirective.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + sortJoinClauses = append(sortJoinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON lt%d.value = ext%d."%s"`, externalTableName, extIndex, labelIndex, extIndex, selectorFieldName)) + //TODO: Verify the field name + sortSelectField = fmt.Sprintf(`ext%d."%s" as ext%d_target`, extIndex, externalFieldName, extIndex) + + } + clause, sortParam, err := buildSortLabelsClause(fields[2], dbName, joinTableIndexByLabelName, sortDirective.Order == sqltypes.ASC) + if err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + orderByClauses = append(orderByClauses, clause) + orderByParams = append(orderByParams, sortParam) + } else if sortDirective.IsIndirect { + if len(sortDirective.IndirectFields) != 4 { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("expected indirect sort directive to have 4 indirect fields, got %d", len(sortDirective.IndirectFields)) + } + externalTableName := getExternalTableName(&sortDirective) + extIndex, ok := joinTableIndexByLabelName[externalTableName] + if !ok { + extIndex = len(joinTableIndexByLabelName) + 1 + joinTableIndexByLabelName[externalTableName] = extIndex + } + selectorFieldName := sortDirective.IndirectFields[2] + if badTableNameChars.MatchString(selectorFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", selectorFieldName) + } + externalFieldName := sortDirective.IndirectFields[3] + if badTableNameChars.MatchString(externalFieldName) { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, fmt.Errorf("invalid database column name '%s'", externalFieldName) + } + columnName := toColumnName(fields) + if err := l.validateColumn(columnName); err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + sortJoinClauses = append(sortJoinClauses, fmt.Sprintf(`JOIN "%s_fields" ext%d ON f."%s" = ext%d."%s"`, externalTableName, extIndex, columnName, extIndex, selectorFieldName)) + direction := "ASC" + nullsPlace := "LAST" + if sortDirective.Order == sqltypes.DESC { + direction = "DESC" + nullsPlace = "FIRST" + } + orderByClauses = append(orderByClauses, fmt.Sprintf(`ext%d."%s" %s NULLS %s`, extIndex, externalFieldName, direction, nullsPlace)) + } else { + columnName := toColumnName(fields) + if err := l.validateColumn(columnName); err != nil { + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, err + } + direction := "ASC" + if sortDirective.Order == sqltypes.DESC { + direction = "DESC" + } + orderByClauses = append(orderByClauses, fmt.Sprintf(`f."%s" %s`, columnName, direction)) + } + } + return sortSelectField, sortJoinClauses, sortWhereClauses, orderByClauses, orderByParams, nil +} + +func (l *ListOptionIndexer) validateColumn(column string) error { + for _, v := range l.indexedFields { + if v == column { + return nil + } + } + return fmt.Errorf("column is invalid [%s]: %w", column, ErrInvalidColumn) +} + +// Helper functions for generating SQL in alphabetical order: + +func applyIndirectLabelTests(loWithLabel *sqltypes.ListOptions, loWithoutLabel *sqltypes.ListOptions, indirectSortDirective *sqltypes.Sort) { + labelFilter := sqltypes.Filter{ + Field: indirectSortDirective.Fields[:], + //Matches: make([]string, 0), + Op: sqltypes.Exists, + } + loWithLabel.Filters = append(loWithLabel.Filters, sqltypes.OrFilter{Filters: []sqltypes.Filter{labelFilter}}) + + // And add an AND-test that the label does not exists for the second test + nonLabelFilter := labelFilter + nonLabelFilter.Op = sqltypes.NotExists + loWithoutLabel.Filters = append(loWithoutLabel.Filters, sqltypes.OrFilter{Filters: []sqltypes.Filter{nonLabelFilter}}) +} + +func buildSortLabelsClause(labelName string, dbName string, joinTableIndexByLabelName map[string]int, isAsc bool) (string, string, error) { + fullName := fmt.Sprintf("%s:%s", dbName, labelName) + labelIndex, ok := joinTableIndexByLabelName[fullName] + if !ok { + return "", "", fmt.Errorf(`internal error: no join-table index given for labelName "%s"`, labelName) + } + stmt := fmt.Sprintf(`CASE lt%d.label WHEN ? THEN lt%d.value ELSE NULL END`, labelIndex, labelIndex) + dir := "ASC" + nullsPosition := "LAST" + if !isAsc { + dir = "DESC" + nullsPosition = "FIRST" + } + return fmt.Sprintf("(%s) %s NULLS %s", stmt, dir, nullsPosition), labelName, nil +} + +func checkForIndirectSortDirective(lo *sqltypes.ListOptions) (*sqltypes.Sort, error) { + indirectSortDirectives := make([]string, 0) + var id *sqltypes.Sort + for _, sd := range lo.SortList.SortDirectives { + if sd.IsIndirect { + id = &sd + indirectSortDirectives = append(indirectSortDirectives, fmt.Sprintf("[%s]", strings.Join(sd.IndirectFields, "]["))) + } + } + if len(indirectSortDirectives) > 1 { + return nil, fmt.Errorf("can have at most one indirect sort directive, have %d: %s", len(indirectSortDirectives), indirectSortDirectives) + } + return id, nil +} + +func formatMatchTarget(filter sqltypes.Filter) string { + format := strictMatchFmt + if filter.Partial { + format = matchFmt + } + return formatMatchTargetWithFormatter(filter.Matches[0], format) +} + +func formatMatchTargetWithFormatter(match string, format string) string { + // To allow matches on the backslash itself, the character needs to be replaced first. + // Otherwise, it will undo the following replacements. + match = strings.ReplaceAll(match, `\`, `\\`) + match = strings.ReplaceAll(match, `_`, `\_`) + match = strings.ReplaceAll(match, `%`, `\%`) + return fmt.Sprintf(format, match) +} + +func getExternalTableName(sd *sqltypes.Sort) string { + s := strings.Join(sd.IndirectFields[0:2], "_") + return strings.ReplaceAll(s, "/", "_") +} + +func isIndirectFilter(filter *sqltypes.Filter) bool { + return filter.IsIndirect +} + +func isLabelFilter(f *sqltypes.Filter) bool { + return len(f.Field) >= 2 && f.Field[0] == "metadata" && f.Field[1] == "labels" +} + +func isLabelsFieldList(fields []string) bool { + return len(fields) == 3 && fields[0] == "metadata" && fields[1] == "labels" +} + +func joinWhereClauses(whereClauses []string, leadingIndent string, continuingIndent string, op string) string { + switch len(whereClauses) { + case 0: + return "" + case 1: + return fmt.Sprintf("%sWHERE %s\n", leadingIndent, whereClauses[0]) + } + separator := fmt.Sprintf(") %s\n%s(", op, continuingIndent) + return fmt.Sprintf("%sWHERE (%s)\n", leadingIndent, strings.Join(whereClauses, separator)) +} + +func prepareComparisonParameters(op sqltypes.Op, target string) (string, float64, error) { + num, err := strconv.ParseFloat(target, 32) + if err != nil { + return "", 0, err + } + switch op { + case sqltypes.Lt: + return "<", num, nil + case sqltypes.Gt: + return ">", num, nil + } + return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op) +} + +func processOrderByFields(sd *sqltypes.Sort, extIndex int, orderByClauses []string) ([]string, []string, []string) { + sortFieldMap := make(map[string]string) + externalFieldName := sd.IndirectFields[3] + newName := fmt.Sprintf("__ix_ext%d_%s", extIndex, nonIdentifierChars.ReplaceAllString(externalFieldName, "_")) + sortFieldMap[externalFieldName] = newName + sortParts := make([]string, 1+len(orderByClauses)) + direction := "ASC" + nullPosition := "LAST" + if sd.Order == sqltypes.DESC { + direction = "DESC" + nullPosition = "FIRST" + } + sortParts[0] = fmt.Sprintf("%s %s NULLS %s", newName, direction, nullPosition) + importWithParts := make([]string, 1+len(orderByClauses)) + importWithParts[0] = fmt.Sprintf(`ext%d."%s" AS %s`, extIndex, externalFieldName, newName) + importAsNullParts := make([]string, 1+len(orderByClauses)) + importAsNullParts[0] = fmt.Sprintf("NULL AS %s", newName) + for i, clause := range orderByClauses { + orderParts := strings.SplitN(clause, " ", 2) + fieldName := orderParts[0] + _, ok := sortFieldMap[fieldName] + if ok { + continue + } + fieldParts := strings.SplitN(fieldName, ".", 2) + prefix := fieldParts[0] + baseName := fieldParts[1] + if baseName[0] == '"' { + baseName = baseName[1 : len(baseName)-1] + } + newBaseName := nonIdentifierChars.ReplaceAllString(baseName, "_") + newName := fmt.Sprintf("__ix_%s_%s", prefix, newBaseName) + sortFieldMap[fieldName] = newName + importWithParts[i+1] = fmt.Sprintf("%s AS %s", fieldName, newName) + importAsNullParts[i+1] = importWithParts[i+1] + sortParts[i+1] = fmt.Sprintf("%s %s", newName, orderParts[1]) + } + return sortParts, importWithParts, importAsNullParts +} + +// There are two kinds of string arrays to turn into a string, based on the last value in the array +// simple: ["a", "b", "conformsToIdentifier"] => "a.b.conformsToIdentifier" +// complex: ["a", "b", "foo.io/stuff"] => "a.b[foo.io/stuff]" + +func smartJoin(s []string) string { + if len(s) == 0 { + return "" + } + if len(s) == 1 { + return s[0] + } + lastBit := s[len(s)-1] + simpleName := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + if simpleName.MatchString(lastBit) { + return strings.Join(s, ".") + } + return fmt.Sprintf("%s[%s]", strings.Join(s[0:len(s)-1], "."), lastBit) +} diff --git a/pkg/sqlcache/informer/query_generator_test.go b/pkg/sqlcache/informer/query_generator_test.go new file mode 100644 index 00000000..ff191edf --- /dev/null +++ b/pkg/sqlcache/informer/query_generator_test.go @@ -0,0 +1,2752 @@ +/* +Copyright 2024, 2025 SUSE LLC +*/ + +package informer + +import ( + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/rancher/steve/pkg/sqlcache/db" + "github.com/rancher/steve/pkg/sqlcache/partition" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestListByOptions(t *testing.T) { + type testCase struct { + description string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + extraIndexedFields []string + expectedList *unstructured.UnstructuredList + returnList []any + expectedContToken string + expectedErr error + } + + testObject := testStoreObject{Id: "something", Val: "a"} + unstrTestObjectMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testObject) + assert.Nil(t, err) + + var tests []testCase + tests = append(tests, testCase{ + description: "ListByOptions() with no errors returned, should not return an error", + listOptions: sqltypes.ListOptions{}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions() with an empty filter, should not return an error", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{{[]sqltypes.Filter{}}}, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ChunkSize: 2}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ?`, + expectedStmtArgs: []interface{}{2}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with Resume set should set offset in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Resume: "4"}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + OFFSET ?`, + expectedStmtArgs: []interface{}{4}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 OrFilter set with 1 filter with Op set top NotEq should select where that filter is not true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" NOT LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 sqltypes.OrFilter set with 1 filter with Partial set to true should select where that partial match on that filter's value is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.somefield" LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with 1 sqltypes.OrFilter set with multiple filters should select where any of those filters are true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"someothervalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"somethirdvalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + ((f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" LIKE ? ESCAPE '\') OR (f."metadata.somefield" NOT LIKE ? ESCAPE '\')) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"%somevalue%", "%someothervalue%", "%somethirdvalue%"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with multiple OrFilters set should select where all OrFilters contain one filter that is true in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"value1"}, + Op: sqltypes.Eq, + Partial: false, + }, + { + Field: []string{"status", "someotherfield"}, + Matches: []string{"value2"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"value3"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test4", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + ((f."metadata.somefield" = ?) OR (f."status.someotherfield" != ?)) AND + (f."metadata.somefield" = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"value1", "value2", "value3", "test4"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with labels filter should select the label in the prepared sql.Stmt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "guard.cattle.io"}, + Matches: []string{"lodgepole"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test41", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"guard.cattle.io", "%lodgepole%", "test41"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with two labels filters should use a self-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"milk"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "horses"}, + Matches: []string{"saddles"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test42", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (lt2.label = ? AND lt2.value = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"cows", "milk", "horses", "saddles", "test42"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with a mix of one label and one non-label query can still self-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"butter"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "somefield"}, + Matches: []string{"wheat"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test43", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (f."metadata.somefield" = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"cows", "butter", "wheat", "test43"}, + returnList: []any{}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{}}, Items: []unstructured.Unstructured{}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with only one sqltypes.Sort.Field set should sort on that field only, in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.somefield" ASC`, + expectedStmtArgs: []any{"test5"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "sort one field descending", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5a", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY f."metadata.somefield" DESC`, + expectedStmtArgs: []any{"test5a"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "sort one unbound label descending", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "flip"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "test5a", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (f."metadata.namespace" = ?) AND + (FALSE) + ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) DESC NULLS FIRST`, + expectedStmtArgs: []any{"flip", "test5a", "flip"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions sorting on two complex fields should sort on the first field in ascending order first and then sort on the second labels field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "fields", "3"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"metadata", "labels", "stub.io/candy"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + extraIndexedFields: []string{"metadata.fields[3]", "metadata.labels[stub.io/candy]"}, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY f."metadata.fields[3]" ASC, (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, + expectedStmtArgs: []any{"stub.io/candy", "stub.io/candy"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions sorting on two fields should sort on the first field in ascending order first and then sort on the second field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"status", "someotherfield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.somefield" ASC, f."status.someotherfield" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions sorting on two fields should sort on the first field in descending order first and then sort on the second field in ascending order in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "somefield"}, + Order: sqltypes.DESC, + }, + { + Fields: []string{"status", "someotherfield"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.somefield" DESC, f."status.someotherfield" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.PageSize set should set limit to PageSize in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + PageSize: 10, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ?`, + expectedStmtArgs: []any{10}, + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.Page and no PageSize set should not add anything to prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + Page: 2, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with sqltypes.Pagination.Page and PageSize set limit to PageSize and offset to PageSize * (Page - 1) in prepared sql.Stmt", + listOptions: sqltypes.ListOptions{ + Pagination: sqltypes.Pagination{ + PageSize: 10, + Page: 2, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE) + ORDER BY f."metadata.name" ASC + LIMIT ? + OFFSET ?`, + expectedStmtArgs: []any{10, 10}, + + expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (FALSE))`, + expectedCountStmtArgs: []any{}, + + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Namespace Partition should select only items where metadata.namespace is equal to Namespace and all other conditions are met in prepared sql.Stmt", + partitions: []partition.Partition{ + { + Namespace: "somens", + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.namespace" = ? AND FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somens"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a All Partition should select all items that meet all other conditions in prepared sql.Stmt", + partitions: []partition.Partition{ + { + All: true, + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Passthrough Partition should select all items that meet all other conditions prepared sql.Stmt", + partitions: []partition.Partition{ + { + Passthrough: true, + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + ORDER BY f."metadata.name" ASC`, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with a Names Partition should select only items where metadata.name equals an items in Names and all other conditions are met in prepared sql.Stmt", + partitions: []partition.Partition{ + { + Names: sets.New[string]("someid", "someotherid"), + }, + }, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.name" IN (?, ?)) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"someid", "someotherid"}, + returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}}, + expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}}, + expectedContToken: "", + expectedErr: nil, + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + txClient := NewMockTXClient(gomock.NewController(t)) + store := NewMockStore(gomock.NewController(t)) + stmts := NewMockStmt(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.somefield", "status.someotherfield"}, + } + if len(test.extraIndexedFields) > 0 { + lii.indexedFields = append(lii.indexedFields, test.extraIndexedFields...) + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") + if test.expectedErr != nil { + assert.Equal(t, test.expectedErr, err) + return + } + assert.Nil(t, err) + assert.Equal(t, test.expectedStmt, queryInfo.query) + if test.expectedStmtArgs == nil { + test.expectedStmtArgs = []any{} + } + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + + stmt := &sql.Stmt{} + rows := &sql.Rows{} + objType := reflect.TypeOf(testObject) + txClient.EXPECT().Stmt(gomock.Any()).Return(stmts).AnyTimes() + store.EXPECT().Prepare(test.expectedStmt).Do(func(a ...any) { + fmt.Println(a) + }).Return(stmt) + if args := test.expectedStmtArgs; args != nil { + stmts.EXPECT().QueryContext(gomock.Any(), gomock.Any()).Return(rows, nil).AnyTimes() + } else if strings.Contains(test.expectedStmt, "LIMIT") { + stmts.EXPECT().QueryContext(gomock.Any(), args...).Return(rows, nil) + txClient.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) + } else { + stmts.EXPECT().QueryContext(gomock.Any()).Return(rows, nil) + } + store.EXPECT().GetType().Return(objType) + store.EXPECT().GetShouldEncrypt().Return(false) + store.EXPECT().ReadObjects(rows, objType, false).Return(test.returnList, nil) + store.EXPECT().CloseStmt(stmt).Return(nil) + + store.EXPECT().WithTransaction(gomock.Any(), false, gomock.Any()).Return(nil).Do( + func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { + err := f(txClient) + if test.expectedErr == nil { + assert.Nil(t, err) + } else { + assert.Equal(t, test.expectedErr, err) + } + }) + + if test.expectedCountStmt != "" { + store.EXPECT().Prepare(test.expectedCountStmt).Return(stmt) + store.EXPECT().ReadInt(rows).Return(len(test.expectedList.Items), nil) + store.EXPECT().CloseStmt(stmt).Return(nil) + } + list, total, contToken, err := lii.executeQuery(context.Background(), queryInfo) + if test.expectedErr == nil { + assert.Nil(t, err) + } else { + assert.Equal(t, test.expectedErr, err) + } + assert.Equal(t, test.expectedList, list) + assert.Equal(t, len(test.expectedList.Items), total) + assert.Equal(t, test.expectedContToken, contToken) + }) + } +} + +func TestConstructQuery(t *testing.T) { + type testCase struct { + description string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr error + } + + var tests []testCase + tests = append(tests, testCase{ + description: "TestConstructQuery: handles IN statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"somevalue"}, + Op: sqltypes.In, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.queryField1" IN (?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOT-IN statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotIn, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + WHERE + (f."metadata.queryField1" NOT IN (?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles EXISTS statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Exists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOT-EXISTS statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.NotExists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedErr: errors.New("NULL and NOT NULL tests aren't supported for non-label queries"), + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualFull"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelEqualFull", "somevalue"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements, match partial", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles != statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNotEqualFull"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNotEqualFull", "labelNotEqualFull", "somevalue"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles != statements for label statements, match partial", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNotEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT LIKE ? ESCAPE '\')) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"labelNotEqualPartial", "labelNotEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles multiple != statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "notEqual1"}, + Matches: []string{"value1"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "notEqual2"}, + Matches: []string{"value2"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt2i1 ON o1.key = lt2i1.key + WHERE lt2i1.label = ?)) OR (lt2.label = ? AND lt2.value != ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"notEqual1", "notEqual1", "value1", "notEqual2", "notEqual2", "value2"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles IN statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelIN"}, + Matches: []string{"somevalue1", "someValue2"}, + Op: sqltypes.In, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value IN (?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelIN", "somevalue1", "someValue2"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOTIN statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNOTIN"}, + Matches: []string{"somevalue1", "someValue2"}, + Op: sqltypes.NotIn, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value NOT IN (?, ?))) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNOTIN", "labelNOTIN", "somevalue1", "someValue2"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles EXISTS statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEXISTS"}, + Matches: []string{}, + Op: sqltypes.Exists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelEXISTS"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles NOTEXISTS statements for label statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelNOTEXISTS"}, + Matches: []string{}, + Op: sqltypes.NotExists, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (o.key NOT IN (SELECT o1.key FROM "something" o1 + JOIN "something_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "something_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"labelNOTEXISTS"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles LessThan statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "numericThing"}, + Matches: []string{"5"}, + Op: sqltypes.Lt, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"numericThing", float64(5)}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "TestConstructQuery: handles GreaterThan statements", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "numericThing"}, + Matches: []string{"35"}, + Op: sqltypes.Gt, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"numericThing", float64(35)}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "multiple filters with a positive label test and a negative non-label test still outer-join", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "junta"}, + Matches: []string{"esther"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"golgi"}, + Op: sqltypes.NotEq, + Partial: true, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"status", "queryField2"}, + Matches: []string{"gold"}, + Op: sqltypes.Eq, + Partial: false, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" NOT LIKE ? ESCAPE '\')) AND + (f."status.queryField2" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"junta", "%esther%", "%golgi%", "gold"}, + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "multiple filters and or-filters with a positive label test and a negative non-label test still outer-join and have correct AND/ORs", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "nectar"}, + Matches: []string{"stash"}, + Op: sqltypes.Eq, + Partial: true, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"landlady"}, + Op: sqltypes.NotEq, + Partial: false, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "lawn"}, + Matches: []string{"reba", "coil"}, + Op: sqltypes.In, + }, + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Gt, + Matches: []string{"2"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') OR (f."metadata.queryField1" != ?)) AND + ((lt2.label = ? AND lt2.value IN (?, ?)) OR (f."metadata.queryField1" > ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, //' + expectedStmtArgs: []any{"nectar", "%stash%", "landlady", "lawn", "reba", "coil", float64(2)}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: handles == statements for label statements, match partial, sort on metadata.queryField1", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "labelEqualPartial"}, + Matches: []string{"somevalue"}, + Op: sqltypes.Eq, + Partial: true, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ? AND lt1.value LIKE ? ESCAPE '\') AND + (FALSE) + ORDER BY f."metadata.queryField1" ASC`, + expectedStmtArgs: []any{"labelEqualPartial", "%somevalue%"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: sort on label statements with no query", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "this"}, + Order: sqltypes.ASC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + WHERE + (lt1.label = ?) AND + (FALSE) + ORDER BY (CASE lt1.label WHEN ? THEN lt1.value ELSE NULL END) ASC NULLS LAST`, //' + expectedStmtArgs: []any{"this", "this"}, + expectedErr: nil, + }) + + tests = append(tests, testCase{ + description: "TestConstructQuery: sort and query on both labels and non-labels without overlap", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"toys"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "labels", "jamb"}, + Matches: []string{"juice"}, + Op: sqltypes.Eq, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "this"}, + Order: sqltypes.ASC, + }, + { + Fields: []string{"status", "queryField2"}, + Order: sqltypes.DESC, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + JOIN "something_fields" f ON o.key = f.key + LEFT OUTER JOIN "something_labels" lt1 ON o.key = lt1.key + LEFT OUTER JOIN "something_labels" lt2 ON o.key = lt2.key + WHERE + ((f."metadata.queryField1" = ?) OR (lt1.label = ? AND lt1.value = ?) OR (lt2.label = ?)) AND + (FALSE) + ORDER BY (CASE lt2.label WHEN ? THEN lt2.value ELSE NULL END) ASC NULLS LAST, f."status.queryField2" DESC`, + expectedStmtArgs: []any{"toys", "jamb", "juice", "this", "this"}, + expectedErr: nil, + }) + + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something") + if test.expectedErr != nil { + assert.Equal(t, test.expectedErr, err) + return + } + require.Nil(t, err) + assert.Equal(t, test.expectedStmt, queryInfo.query) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestSmartJoin(t *testing.T) { + type testCase struct { + description string + fieldArray []string + expectedFieldName string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "single-letter names should be dotted", + fieldArray: []string{"metadata", "labels", "a"}, + expectedFieldName: "metadata.labels.a", + }) + tests = append(tests, testCase{ + description: "underscore should be dotted", + fieldArray: []string{"metadata", "labels", "_"}, + expectedFieldName: "metadata.labels._", + }) + tests = append(tests, testCase{ + description: "simple names should be dotted", + fieldArray: []string{"metadata", "labels", "queryField2"}, + expectedFieldName: "metadata.labels.queryField2", + }) + tests = append(tests, testCase{ + description: "a numeric field should be bracketed", + fieldArray: []string{"metadata", "fields", "43"}, + expectedFieldName: "metadata.fields[43]", + }) + tests = append(tests, testCase{ + description: "a field starting with a number should be bracketed", + fieldArray: []string{"metadata", "fields", "46days"}, + expectedFieldName: "metadata.fields[46days]", + }) + tests = append(tests, testCase{ + description: "compound names should be bracketed", + fieldArray: []string{"metadata", "labels", "rancher.cattle.io/moo"}, + expectedFieldName: "metadata.labels[rancher.cattle.io/moo]", + }) + tests = append(tests, testCase{ + description: "space-separated names should be bracketed", + fieldArray: []string{"metadata", "labels", "space here"}, + expectedFieldName: "metadata.labels[space here]", + }) + tests = append(tests, testCase{ + description: "already-bracketed terms cause double-bracketing and should never be used", + fieldArray: []string{"metadata", "labels[k8s.io/deepcode]"}, + expectedFieldName: "metadata[labels[k8s.io/deepcode]]", + }) + tests = append(tests, testCase{ + description: "an empty array should be an empty string", + fieldArray: []string{}, + expectedFieldName: "", + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + gotFieldName := smartJoin(test.fieldArray) + assert.Equal(t, test.expectedFieldName, gotFieldName) + }) + } +} + +func TestBuildSortLabelsClause(t *testing.T) { + type testCase struct { + description string + labelName string + joinTableIndexByLabelName map[string]int + direction bool + expectedStmt string + expectedParam string + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "TestBuildSortClause: empty index list errors", + labelName: "emptyListError", + expectedErr: `internal error: no join-table index given for labelName "emptyListError"`, + }) + tests = append(tests, testCase{ + description: "TestBuildSortClause: hit ascending", + labelName: "testBSL1", + joinTableIndexByLabelName: map[string]int{"db:testBSL1": 3}, + direction: true, + expectedStmt: `(CASE lt3.label WHEN ? THEN lt3.value ELSE NULL END) ASC NULLS LAST`, + expectedParam: "testBSL1", + }) + tests = append(tests, testCase{ + description: "TestBuildSortClause: hit descending", + labelName: "testBSL2", + joinTableIndexByLabelName: map[string]int{"db:testBSL2": 4}, + direction: false, + expectedStmt: `(CASE lt4.label WHEN ? THEN lt4.value ELSE NULL END) DESC NULLS FIRST`, + expectedParam: "testBSL2", + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + stmt, param, err := buildSortLabelsClause(test.labelName, "db", test.joinTableIndexByLabelName, test.direction) + if test.expectedErr != "" { + assert.Equal(t, test.expectedErr, err.Error()) + } else { + assert.Nil(t, err) + assert.Equal(t, test.expectedStmt, stmt) + assert.Equal(t, test.expectedParam, param) + } + }) + } +} + +func TestConstructIndirectLabelFilterQuery(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.NotEq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.NotEq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" != ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Lt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"10"}, + Op: sqltypes.Lt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", float64(10)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Gt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"11"}, + Op: sqltypes.Gt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", float64(11)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Op: sqltypes.Exists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" != NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect Not-sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Op: sqltypes.NotExists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" == NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.In-Set", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"fish", "cows", "ships"}, + Op: sqltypes.In, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "fish", "cows", "ships"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery: simple redirect sqltypes.NotIn", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"balloons", "clubs", "cheese"}, + Op: sqltypes.NotIn, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE (lt1.label = ? AND ext2."spec.displayName" NOT IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "balloons", "clubs", "cheese"}, + expectedErr: "", + }) + // There's no easy way to safely parameterize column names, as opposed to values, + // so verify that the code generator checked them for whitelisted characters. + // Allow only [-a-zA-Z0-9$_\[\].]+ + tests = append(tests, testCase{ + description: "IndirectFilterQuery: verify the injected field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "name ; drop database marks ; select * from _v1_Namespace", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'name ; drop database marks ; select * from _v1_Namespace'", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + assert.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } + +} + +func TestConstructIndirectNonLabelFilterQuery(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" = ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.NotEq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.NotEq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" != ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"System"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Lt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"10"}, + Op: sqltypes.Lt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" < ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{float64(10)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Gt", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"11"}, + Op: sqltypes.Gt, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" > ?) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{float64(11)}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.Exists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" != NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect Not-sqltypes.Exists", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Op: sqltypes.NotExists, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" == NULL) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.In-Set", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"fish", "cows", "ships"}, + Op: sqltypes.In, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"fish", "cows", "ships"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: simple redirect sqltypes.NotIn", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"balloons", "clubs", "cheese"}, + Op: sqltypes.NotIn, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (ext1."spec.displayName" NOT IN (?, ?, ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"balloons", "clubs", "cheese"}, + expectedErr: "", + }) + // There's no easy way to safely parameterize column names, as opposed to values, + // so verify that the code generator checked them for whitelisted characters. + // Allow only [-a-zA-Z0-9$_\[\].]+ + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: verify the injected external field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "name ; drop database marks ; select * from _v1_Namespace", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'name ; drop database marks ; select * from _v1_Namespace'", + }) + tests = append(tests, testCase{ + description: "IndirectFilterQuery - no label: verify the injected selecting field name is safe", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "queryField1 ; drop database thought-this-is=-checked"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "projects", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "column is invalid [metadata[queryField1 ; drop database thought-this-is=-checked]]: supplied column is invalid", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructMixedLabelIndirect(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - one label, one redirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "radio"}, + Matches: []string{"fish"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.displayName" = ?)) AND + (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"radio", "fish", "System"}, + expectedErr: "", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructMixedMultiTypes(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "IndirectFilterQuery - mix of label,non-label x direct,indirect sqltypes.Eq", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "suitcase"}, + Matches: []string{"valid"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"sprint"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.displayName"}, + }, + { + Field: []string{"status", "queryField2"}, + Matches: []string{"moisture"}, + Op: sqltypes.Eq, + }, + }, + }, + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "green"}, + Matches: []string{"squalor"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"tournaments.cattle.io/v3", "Diary", "metadata.name", "spocks.brain"}, + }, + }, + }, + }}, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.name" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + JOIN "tournaments.cattle.io_v3_Diary_fields" ext4 ON lt3.value = ext4."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) + OR (ext2."spec.displayName" = ?) + OR (f."status.queryField2" = ?)) + AND (lt3.label = ? AND ext4."spocks.brain" = ?) + AND (FALSE) + ORDER BY f."metadata.name" ASC`, + expectedStmtArgs: []any{"suitcase", "valid", "sprint", "moisture", "green", "squalor"}, + expectedErr: "", + }) + + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery) + assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams) + }) + } +} + +func TestConstructLabelIndirectSort(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, no filters, happy path", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM ( + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, ext2."spec.clusterName" AS __ix_ext2_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ext2 ON lt1.value = ext2."metadata.name" + WHERE lt1.label = ? +UNION ALL + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, NULL AS __ix_ext2_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + WHERE o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1 + JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1i1 ON o1.key = lt1i1.key + WHERE lt1i1.label = ?) +) +WHERE FALSE + ORDER BY __ix_ext2_spec_clusterName ASC NULLS LAST, __ix_f_metadata_name ASC`, + expectedStmtArgs: []any{"field.cattle.io/projectId", "field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, two filters, happy path", + listOptions: sqltypes.ListOptions{ + Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "radio"}, + Matches: []string{"camels"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "queryField1"}, + Matches: []string{"System"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"tournaments.cattle.io/v3", "Capsule", "metadata.namespace", "spec.heights"}, + }, + }, + }, + }, + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT __ix_object, __ix_objectnonce, __ix_dekid FROM ( + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, ext4."spec.clusterName" AS __ix_ext4_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "tournaments.cattle.io_v3_Capsule_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.namespace" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + JOIN "management.cattle.io_v3_Project_fields" ext4 ON lt3.value = ext4."metadata.name" + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.heights" = ?)) AND (lt3.label = ?) +UNION ALL + SELECT DISTINCT o.object AS __ix_object, o.objectnonce AS __ix_objectnonce, o.dekid AS __ix_dekid, NULL AS __ix_ext4_spec_clusterName, f."metadata.name" AS __ix_f_metadata_name FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON o.key = lt1.key + JOIN "tournaments.cattle.io_v3_Capsule_fields" ext2 ON f."metadata.queryField1" = ext2."metadata.namespace" + LEFT OUTER JOIN "_v1_Namespace_labels" lt3 ON o.key = lt3.key + WHERE ((lt1.label = ? AND lt1.value = ?) OR (ext2."spec.heights" = ?)) AND (o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1 + JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key + LEFT OUTER JOIN "_v1_Namespace_labels" lt3i1 ON o1.key = lt3i1.key + WHERE lt3i1.label = ?)) +) +WHERE FALSE + ORDER BY __ix_ext4_spec_clusterName ASC NULLS LAST, __ix_f_metadata_name ASC`, + expectedStmtArgs: []any{"radio", "camels", "System", "field.cattle.io/projectId", "radio", "camels", "System", "field.cattle.io/projectId"}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "foo; drop database bobby1", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'foo; drop database bobby1'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "bar; drop database bobby2"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'bar; drop database bobby2'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, not enough indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 3", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, too many indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "field.cattle.io/projectId"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "little", "bobby-tables"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 5", + }) + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + }) + } +} + +func TestConstructSimpleNonLabelIndirectSort(t *testing.T) { + type testCase struct { + description string + dbname string + listOptions sqltypes.ListOptions + partitions []partition.Partition + ns string + expectedCountStmt string + expectedCountStmtArgs []any + expectedStmt string + expectedStmtArgs []any + expectedErr string + } + + var tests []testCase + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one sort, no filters, no labels", + // Find all mcio.Projects that have the same metadata.name as the namespace, and sort by associated spec.clusterName + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM + "_v1_Namespace" o + JOIN "_v1_Namespace_fields" f ON o.key = f.key + JOIN "management.cattle.io_v3_Project_fields" ext1 ON f."metadata.queryField1" = ext1."metadata.name" + WHERE (FALSE) + ORDER BY ext1."spec.clusterName" ASC NULLS LAST`, + expectedStmtArgs: []any{}, + expectedErr: "", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "foo; drop database bobby1", "spec.clusterName"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'foo; drop database bobby1'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, invalid external selector-column", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "bar; drop database bobby2"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "invalid database column name 'bar; drop database bobby2'", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, not enough indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 3", + }) + tests = append(tests, testCase{ + description: "SimpleIndirectSort - one non-label sort, too many indirect fields", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "queryField1"}, + Order: sqltypes.ASC, + IsIndirect: true, + IndirectFields: []string{"management.cattle.io/v3", "Project", "metadata.name", "little", "bobby-tables"}, + }, + }, + }, + }, + partitions: []partition.Partition{}, + ns: "", + dbname: "_v1_Namespace", + expectedErr: "expected indirect sort directive to have 4 indirect fields, got 5", + }) + t.Parallel() + ptn := regexp.MustCompile(`\s+`) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + store := NewMockStore(gomock.NewController(t)) + i := &Indexer{ + Store: store, + } + lii := &ListOptionIndexer{ + Indexer: i, + indexedFields: []string{"metadata.queryField1", "status.queryField2"}, + } + dbname := test.dbname + if dbname == "" { + dbname = "sometable" + } + queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, dbname) + if test.expectedErr != "" { + require.NotNil(t, err) + assert.Equal(t, test.expectedErr, err.Error()) + return + } + require.Nil(t, err) + expectedStmt := ptn.ReplaceAllString(strings.TrimSpace(test.expectedStmt), " ") + receivedStmt := ptn.ReplaceAllString(strings.TrimSpace(queryInfo.query), " ") + assert.Equal(t, expectedStmt, receivedStmt) + assert.Equal(t, test.expectedStmtArgs, queryInfo.params) + }) + } +} diff --git a/pkg/sqlcache/sqltypes/types.go b/pkg/sqlcache/sqltypes/types.go index eecea863..6fae8d7b 100644 --- a/pkg/sqlcache/sqltypes/types.go +++ b/pkg/sqlcache/sqltypes/types.go @@ -25,11 +25,11 @@ const ( // ListOptions represents the query parameters that may be included in a list request. type ListOptions struct { - ChunkSize int - Resume string - Filters []OrFilter - SortList SortList - Pagination Pagination + ChunkSize int `json:"chunkSize"` + Resume string `json:"resume"` + Filters []OrFilter `json:"orFilters"` + SortList SortList `json:"sortList"` + Pagination Pagination `json:"pagination"` } // Filter represents a field to filter by. @@ -40,12 +40,12 @@ type ListOptions struct { // // If more than one value is given for the `Match` field, we do an "IN ()" test type Filter struct { - Field []string - Matches []string - Op Op - Partial bool - IsIndirect bool - IndirectFields []string + Field []string `json:"fields"` + Matches []string `json:"matches"` + Op Op `json:"op"` + Partial bool `json:"partial"` + IsIndirect bool `json:"isIndirect"` + IndirectFields []string `json:"indirectFields"` } // OrFilter represents a set of possible fields to filter by, where an item may match any filter in the set to be included in the result. @@ -59,20 +59,20 @@ type OrFilter struct { // The order is represented by prefixing the sort key by '-', e.g. sort=-metadata.name. // e.g. To sort internal clusters first followed by clusters in alpha order: sort=-spec.internal,spec.displayName type Sort struct { - Fields []string - Order SortOrder - IsIndirect bool - IndirectFields []string + Fields []string `json:"fields"` + Order SortOrder `json:"order"` + IsIndirect bool `json:"isIndirect"` + IndirectFields []string `json:"indirectFields"` } type SortList struct { - SortDirectives []Sort + SortDirectives []Sort `json:"sortDirectives"` } // Pagination represents how to return paginated results. type Pagination struct { - PageSize int - Page int + PageSize int `json:"pageSize"` + Page int `json:"page"` } func NewSortList() *SortList { diff --git a/pkg/stores/sqlpartition/listprocessor/processor.go b/pkg/stores/sqlpartition/listprocessor/processor.go index 7afd54e1..1e6813a3 100644 --- a/pkg/stores/sqlpartition/listprocessor/processor.go +++ b/pkg/stores/sqlpartition/listprocessor/processor.go @@ -73,11 +73,14 @@ func k8sRequirementToOrFilter(requirement queryparser.Requirement) (sqltypes.Fil values := requirement.Values() queryFields := splitQuery(requirement.Key()) op, usePartialMatch, err := k8sOpToRancherOp(requirement.Operator()) + isIndirect, indirectFields := requirement.IndirectInfo() return sqltypes.Filter{ - Field: queryFields, - Matches: values, - Op: op, - Partial: usePartialMatch, + Field: queryFields, + Matches: values, + Op: op, + Partial: usePartialMatch, + IsIndirect: isIndirect, + IndirectFields: indirectFields, }, err } diff --git a/pkg/stores/sqlpartition/listprocessor/processor_test.go b/pkg/stores/sqlpartition/listprocessor/processor_test.go index bc912cc1..92ba8af2 100644 --- a/pkg/stores/sqlpartition/listprocessor/processor_test.go +++ b/pkg/stores/sqlpartition/listprocessor/processor_test.go @@ -370,6 +370,33 @@ func TestParseQuery(t *testing.T) { }, }, }) + tests = append(tests, testCase{ + description: "ParseQuery() with an indirect labels filter param should create an indirect labels-specific filter.", + req: &types.APIRequest{ + Request: &http.Request{ + URL: &url.URL{RawQuery: "filter=metadata.labels[grover.example.com/fish]=>[_v1][Foods][foodCode][country]=japan"}, + }, + }, + expectedLO: sqltypes.ListOptions{ + ChunkSize: defaultLimit, + Filters: []sqltypes.OrFilter{ + { + Filters: []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "grover.example.com/fish"}, + Matches: []string{"japan"}, + Op: sqltypes.Eq, + IsIndirect: true, + IndirectFields: []string{"_v1", "Foods", "foodCode", "country"}, + }, + }, + }, + }, + Pagination: sqltypes.Pagination{ + Page: 1, + }, + }, + }) tests = append(tests, testCase{ description: "ParseQuery() with multiple filter params, should include multiple or filters.", req: &types.APIRequest{