Merge ca28cd31be into 392a95f753

pkg/sqlcache/informer/listoption_indexer.go

@@ -248,11 +248,18 @@ func (l *ListOptionIndexer) deleteLabels(key string, tx transaction.Client) erro
 //   - a continue token, if there are more pages after the returned one
 //   - an error instead of all of the above if anything went wrong
 func (l *ListOptionIndexer) ListByOptions(ctx context.Context, lo ListOptions, partitions []partition.Partition, namespace string) (*unstructured.UnstructuredList, int, string, error) {
-	queryInfo, err := l.constructQuery(lo, partitions, namespace, db.Sanitize(l.GetName()))
+	dbName := db.Sanitize(l.GetName())
+	queryInfo, err := l.constructQuery(lo, partitions, namespace, dbName)
 	if err != nil {
 		return nil, 0, "", err
 	}
-	return l.executeQuery(ctx, queryInfo)
+	list, total, token, err := l.executeQuery(ctx, queryInfo)
+	if err != nil {
+		if isSpecialCaseQuery(lo, namespace, dbName) && removeSpecialCaseSituation(&lo, namespace, dbName) {
+			return l.ListByOptions(ctx, lo, partitions, namespace)
+		}
+	}
+	return list, total, token, err
 }
 
 // QueryInfo is a helper-struct that is used to represent the core query and parameters when converting
@@ -266,11 +273,9 @@ type QueryInfo struct {
 	offset      int
 }
 
-func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) {
+func (l *ListOptionIndexer) constructRegularQuery(lo ListOptions, namespace string, dbName string, joinTableIndexByLabelName map[string]int) (string, []string, []any, error) {
 	ensureSortLabelsAreSelected(&lo)
-	queryInfo := &QueryInfo{}
 	queryUsesLabels := hasLabelFilter(lo.Filters)
-	joinTableIndexByLabelName := make(map[string]int)
 
 	// First, what kind of filtering will we be doing?
 	// 1- Intro: SELECT and JOIN clauses
@@ -308,7 +313,7 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 	for _, orFilters := range lo.Filters {
 		orClause, orParams, err := l.buildORClauseFromFilters(orFilters, dbName, joinTableIndexByLabelName)
 		if err != nil {
-			return queryInfo, err
+			return query, whereClauses, params, err
 		}
 		if orClause == "" {
 			continue
@@ -322,6 +327,60 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 		whereClauses = append(whereClauses, fmt.Sprintf(`f."metadata.namespace" = ?`))
 		params = append(params, namespace)
 	}
+	return query, whereClauses, params, nil
+}
+
+func constructSpecialQuery(lo ListOptions, namespace string, dbName string) (string, []string, []string, error) {
+	if dbName != "_v1_Namespace" {
+		return "", nil, nil, fmt.Errorf("internal error: dbName must be %s, got %s", "_v1_Namespace", dbName)
+	}
+	// If we're grabbing all the namespaces with no filter or sort parameters,
+	// we want them sorted first according to the human name of the project they're associated with,
+	// if there is one, and then sort by namespace name.
+	// Namespaces that aren't in a project show up after the others.
+	// Note that this query will fail if the 'management.cattle.io_v3_Project*' tables haven't
+	// been loaded yet. In which case we redo the query with a sort on `metadata.name`
+	// to prevent this query from being created.
+	query := `SELECT object, objectnonce, dekid FROM
+(
+  SELECT o.object as object, o.objectnonce as objectnonce, o.dekid as dekid, o.key as key, proj."spec.displayName" as humanName FROM "_v1_Namespace" o
+    JOIN "_v1_Namespace_fields" f ON o.key = f.key
+    LEFT OUTER JOIN "_v1_Namespace_labels" nslb ON o.key = nslb.key 
+    JOIN "management.cattle.io_v3_Project_fields" proj ON nslb.value = proj."metadata.name"
+    WHERE nslb.label = "field.cattle.io/projectId"
+
+  UNION ALL
+
+    SELECT o.object as object, o.objectnonce as objectnonce, o.dekid as dekid, o.key as key, NULL as humanName FROM "_v1_Namespace" o
+    JOIN "_v1_Namespace_fields" f ON o.key = f.key
+    LEFT OUTER JOIN "_v1_Namespace_labels" nslb ON o.key = nslb.key
+    WHERE (o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1
+           JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key
+           LEFT OUTER JOIN "_v1_Namespace_labels" lt1i1 ON o1.key = lt1i1.key
+           WHERE lt1i1.label = "field.cattle.io/projectId"))
+ )`
+	whereClauses := []string{}
+	sortClauses := []string{"humanName ASC NULLS LAST", "key ASC"}
+	return query, whereClauses, sortClauses, nil
+}
+
+func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partition.Partition, namespace string, dbName string) (*QueryInfo, error) {
+	joinTableIndexByLabelName := make(map[string]int)
+	var err error
+	var query string
+	var whereClauses []string
+	var orderByClauses []string
+	var params []any
+	if isSpecialCaseQuery(lo, namespace, dbName) {
+		query, whereClauses, orderByClauses, err = constructSpecialQuery(lo, namespace, dbName)
+		params = make([]any, 0)
+	} else {
+		query, whereClauses, params, err = l.constructRegularQuery(lo, namespace, dbName, joinTableIndexByLabelName)
+		orderByClauses = make([]string, 0)
+	}
+	if err != nil {
+		return nil, err
+	}
 
 	// WHERE clauses (from partitions and their corresponding parameters)
 	partitionClauses := []string{}
@@ -341,7 +400,7 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 			if !thisPartition.All {
 				names := thisPartition.Names
 
-				if len(names) == 0 {
+				if names.Len() == 0 {
 					// degenerate case, there will be no results
 					singlePartitionClauses = append(singlePartitionClauses, "FALSE")
 				} else {
@@ -388,11 +447,14 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 	countParams := params[:]
 
 	// 3- Sorting: ORDER BY clauses (from lo.Sort)
-	if len(lo.Sort.Fields) != len(lo.Sort.Orders) {
+	if len(orderByClauses) > 0 {
+		// From the special-case query
+		query += "\n  ORDER BY "
+		query += strings.Join(orderByClauses, ", ")
+	} else if len(lo.Sort.Fields) != len(lo.Sort.Orders) {
 		return nil, fmt.Errorf("sort fields length %d != sort orders length %d", len(lo.Sort.Fields), len(lo.Sort.Orders))
-	}
-	if len(lo.Sort.Fields) > 0 {
-		orderByClauses := []string{}
+	} else if len(lo.Sort.Fields) > 0 {
+		orderByClauses = []string{}
 		for i, field := range lo.Sort.Fields {
 			if isLabelsFieldList(field) {
 				clause, sortParam, err := buildSortLabelsClause(field[2], joinTableIndexByLabelName, lo.Sort.Orders[i] == ASC)
@@ -404,7 +466,7 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 			} else {
 				columnName := toColumnName(field)
 				if err := l.validateColumn(columnName); err != nil {
-					return queryInfo, err
+					return nil, err
 				}
 				direction := "ASC"
 				if lo.Sort.Orders[i] == DESC {
@@ -443,7 +505,7 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 	if lo.Resume != "" {
 		offsetInt, err := strconv.Atoi(lo.Resume)
 		if err != nil {
-			return queryInfo, err
+			return nil, err
 		}
 		offset = offsetInt
 	}
@@ -454,6 +516,8 @@ func (l *ListOptionIndexer) constructQuery(lo ListOptions, partitions []partitio
 		offsetClause = "\n  OFFSET ?"
 		params = append(params, offset)
 	}
+
+	queryInfo := &QueryInfo{}
 	if limit > 0 || offset > 0 {
 		query += limitClause
 		query += offsetClause
@@ -807,20 +871,6 @@ func (l *ListOptionIndexer) getLabelFilter(index int, filter Filter, dbName stri
 	return "", nil, fmt.Errorf("unrecognized operator: %s", opString)
 }
 
-func prepareComparisonParameters(op Op, target string) (string, float64, error) {
-	num, err := strconv.ParseFloat(target, 32)
-	if err != nil {
-		return "", 0, err
-	}
-	switch op {
-	case Lt:
-		return "<", num, nil
-	case Gt:
-		return ">", num, nil
-	}
-	return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op)
-}
-
 func formatMatchTarget(filter Filter) string {
 	format := strictMatchFmt
 	if filter.Partial {
@@ -838,6 +888,37 @@ func formatMatchTargetWithFormatter(match string, format string) string {
 	return fmt.Sprintf(format, match)
 }
 
+func isSpecialCaseQuery(lo ListOptions, namespace string, dbName string) bool {
+	if dbName == "_v1_Namespace" {
+		return (namespace == "" || namespace == "*") && len(lo.Filters) == 0 && len(lo.Sort.Fields) == 0
+	}
+	return false
+}
+
+func prepareComparisonParameters(op Op, target string) (string, float64, error) {
+	num, err := strconv.ParseFloat(target, 32)
+	if err != nil {
+		return "", 0, err
+	}
+	switch op {
+	case Lt:
+		return "<", num, nil
+	case Gt:
+		return ">", num, nil
+	}
+	return "", 0, fmt.Errorf("unrecognized operator when expecting '<' or '>': '%s'", op)
+}
+
+func removeSpecialCaseSituation(lo *ListOptions, namespace string, dbName string) bool {
+	if dbName == "_v1_Namespace" {
+		// Muddy it and retry
+		lo.Sort.Fields = [][]string{{"metadata", "name"}}
+		lo.Sort.Orders = []SortOrder{ASC}
+		return true
+	}
+	return false
+}
+
 // There are two kinds of string arrays to turn into a string, based on the last value in the array
 // simple: ["a", "b", "conformsToIdentifier"] => "a.b.conformsToIdentifier"
 // complex: ["a", "b", "foo.io/stuff"] => "a.b[foo.io/stuff]"

pkg/sqlcache/informer/listoption_indexer_test.go

@@ -1025,6 +1025,7 @@ func TestListByOptions(t *testing.T) {
 	}
 }
 
+// `'
 func TestConstructQuery(t *testing.T) {
 	type testCase struct {
 		description           string
@@ -1679,6 +1680,138 @@ func TestConstructQuery(t *testing.T) {
 	}
 }
 
+func TestConstructSpecialQuery(t *testing.T) {
+	type testCase struct {
+		dbname                string
+		description           string
+		listOptions           ListOptions
+		partitions            []partition.Partition
+		ns                    string
+		expectedCountStmt     string
+		expectedCountStmtArgs []any
+		expectedStmt          string
+		expectedStmtArgs      []any
+		expectedErr           error
+	}
+
+	var tests []testCase
+	tests = append(tests, testCase{
+		dbname:      "_v1_Namespace",
+		description: "ConstructQuery: unsorted namespaces should be sorted by project name",
+		listOptions: ListOptions{},
+		partitions:  []partition.Partition{},
+		ns:          "",
+		expectedStmt: `SELECT object, objectnonce, dekid FROM
+(
+  SELECT o.object as object, o.objectnonce as objectnonce, o.dekid as dekid, o.key as key, proj."spec.displayName" as humanName FROM "_v1_Namespace" o
+    JOIN "_v1_Namespace_fields" f ON o.key = f.key
+    LEFT OUTER JOIN "_v1_Namespace_labels" nslb ON o.key = nslb.key 
+    JOIN "management.cattle.io_v3_Project_fields" proj ON nslb.value = proj."metadata.name"
+    WHERE nslb.label = "field.cattle.io/projectId"
+
+  UNION ALL
+
+    SELECT o.object as object, o.objectnonce as objectnonce, o.dekid as dekid, o.key as key, NULL as humanName FROM "_v1_Namespace" o
+    JOIN "_v1_Namespace_fields" f ON o.key = f.key
+    LEFT OUTER JOIN "_v1_Namespace_labels" nslb ON o.key = nslb.key
+    WHERE (o.key NOT IN (SELECT o1.key FROM "_v1_Namespace" o1
+           JOIN "_v1_Namespace_fields" f1 ON o1.key = f1.key
+           LEFT OUTER JOIN "_v1_Namespace_labels" lt1i1 ON o1.key = lt1i1.key
+           WHERE lt1i1.label = "field.cattle.io/projectId"))
+ )
+  WHERE
+    (FALSE)
+  ORDER BY humanName ASC NULLS LAST, key ASC`,
+		expectedStmtArgs: []any{},
+		expectedErr:      nil,
+	})
+	t.Parallel()
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			store := NewMockStore(gomock.NewController(t))
+			i := &Indexer{
+				Store: store,
+			}
+			lii := &ListOptionIndexer{
+				Indexer:       i,
+				indexedFields: []string{"metadata.queryField1", "status.queryField2"},
+			}
+			if test.listOptions.Filters == nil {
+				test.listOptions.Filters = []OrFilter{}
+			}
+			if test.listOptions.Sort.Fields == nil {
+				test.listOptions.Sort.Fields = [][]string{}
+				test.listOptions.Sort.Orders = []SortOrder{}
+			}
+			assert.True(t, isSpecialCaseQuery(test.listOptions, test.ns, test.dbname))
+			queryInfo, err := lii.constructQuery(test.listOptions, test.partitions, test.ns, test.dbname)
+			if test.expectedErr != nil {
+				assert.Equal(t, test.expectedErr, err)
+				return
+			}
+			assert.Nil(t, err)
+			assert.Equal(t, test.expectedStmt, queryInfo.query)
+			assert.Equal(t, test.expectedStmtArgs, queryInfo.params)
+			assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery)
+			assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams)
+		})
+	}
+}
+
+func TestRemoveSpecialCaseOnNamespace(t *testing.T) {
+	type testCase struct {
+		dbname                string
+		description           string
+		listOptions           ListOptions
+		partitions            []partition.Partition
+		ns                    string
+		expectedCountStmt     string
+		expectedCountStmtArgs []any
+		expectedStmt          string
+		expectedStmtArgs      []any
+		expectedErr           error
+	}
+
+	test := testCase{
+		dbname:      "_v1_Namespace",
+		description: "RemoveSpecialCaseOnNamespace: unsorted namespaces should be sorted by namespace name",
+		listOptions: ListOptions{
+			Filters: []OrFilter{},
+			Sort:    Sort{Fields: [][]string{}, Orders: []SortOrder{}},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "_v1_Namespace" o
+  JOIN "_v1_Namespace_fields" f ON o.key = f.key
+  WHERE
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{},
+		expectedErr:      nil,
+	}
+	assert.True(t, isSpecialCaseQuery(test.listOptions, test.ns, test.dbname))
+	assert.True(t, removeSpecialCaseSituation(&test.listOptions, test.ns, test.dbname))
+	assert.False(t, isSpecialCaseQuery(test.listOptions, test.ns, test.dbname))
+	store := NewMockStore(gomock.NewController(t))
+	i := &Indexer{
+		Store: store,
+	}
+	lii := &ListOptionIndexer{
+		Indexer:       i,
+		indexedFields: []string{"metadata.name"},
+	}
+	queryInfo, err := lii.constructQuery(test.listOptions, test.partitions, test.ns, test.dbname)
+	if test.expectedErr != nil {
+		assert.Equal(t, test.expectedErr, err)
+		return
+	}
+	assert.Nil(t, err)
+	assert.Equal(t, test.expectedStmt, queryInfo.query)
+	assert.Equal(t, test.expectedStmtArgs, queryInfo.params)
+	assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery)
+	assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams)
+}
+
 func TestSmartJoin(t *testing.T) {
 	type testCase struct {
 		description       string
@@ -1748,7 +1881,7 @@ func TestBuildSortLabelsClause(t *testing.T) {
 		joinTableIndexByLabelName map[string]int
 		direction                 bool
 		expectedStmt              string
-        expectedParam             string
+		expectedParam             string
 		expectedErr               string
 	}
 

pkg/stores/sqlproxy/proxy_store.go

@@ -151,7 +151,9 @@ var (
 		gvkKey("management.cattle.io", "v3", "NodeTemplate"): {
 			{"spec", "clusterName"}},
 		gvkKey("management.cattle.io", "v3", "Project"): {
-			{"spec", "clusterName"}},
+			{"spec", "clusterName"},
+			{"spec", "displayName"},
+		},
 		gvkKey("networking.k8s.io", "v1", "Ingress"): {
 			{"spec", "rules", "host"},
 			{"spec", "ingressClassName"},