[v2.9] Backport PR 271: Index more sqlite cache fields. (#315)

* [v2.9] Backport PR 271: Index more sqlite cache fields.

* go mod tidy

Signed-off-by: Silvio Moioli <silvio@moioli.net>

---------

Signed-off-by: Silvio Moioli <silvio@moioli.net>
Co-authored-by: Silvio Moioli <silvio@moioli.net>

main.go

@@ -34,7 +34,7 @@ func main() {
 func run(_ *cli.Context) error {
 	ctx := signals.SetupSignalContext()
 	debugconfig.MustSetupDebug()
-	s, err := config.ToServer(ctx, false)
+	s, err := config.ToServer(ctx, debugconfig.SQLCache)
 	if err != nil {
 		return err
 	}

pkg/debug/cli.go

@@ -13,6 +13,7 @@ import (
 type Config struct {
 	Debug      bool
 	DebugLevel int
+	SQLCache   bool
 }
 
 func (c *Config) MustSetupDebug() {
@@ -54,6 +55,10 @@ func Flags(config *Config) []cli.Flag {
 			Value:       7,
 			Destination: &config.DebugLevel,
 		},
+		cli.BoolFlag{
+			Name:        "sql-cache",
+			Destination: &config.SQLCache,
+		},
 	}
 }
 
@@ -68,5 +73,9 @@ func FlagsV2(config *Config) []cliv2.Flag {
 			Value:       7,
 			Destination: &config.DebugLevel,
 		},
+		&cliv2.BoolFlag{
+			Name:        "sql-cache",
+			Destination: &config.SQLCache,
+		},
 	}
 }

pkg/resources/virtual/common/common.go

@@ -10,7 +10,6 @@ import (
 	wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/tools/cache"
 )
 
 // SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache
@@ -24,26 +23,14 @@ type DefaultFields struct {
 	Cache SummaryCache
 }
 
-// GetTransform produces the default transformation func
-func (d *DefaultFields) GetTransform() cache.TransformFunc {
-	return d.transform
-}
-
-// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary
-func (d *DefaultFields) transform(obj any) (any, error) {
-	raw, isSignal, err := getUnstructured(obj)
-	if isSignal {
-		return obj, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	raw = addIDField(raw)
-	raw, err = addSummaryFields(raw, d.Cache)
+// TransformCommon implements virtual.VirtualTransformFunc, and adds reserved fields/summary
+func (d *DefaultFields) TransformCommon(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
+	obj = addIDField(obj)
+	obj, err := addSummaryFields(obj, d.Cache)
 	if err != nil {
 		return nil, fmt.Errorf("unable to add summary fields: %w", err)
 	}
-	return raw, nil
+	return obj, nil
 }
 
 // addSummaryFields adds the virtual fields for object state.

pkg/resources/virtual/common/common_test.go

@@ -9,11 +9,10 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/cache"
 )
 
-func TestTransform(t *testing.T) {
+func TestTransformCommonObjects(t *testing.T) {
 	tests := []struct {
 		name             string
 		input            any
@@ -160,29 +159,29 @@ func TestTransform(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			fakeCache := fakeSummaryCache{
-				summarizedObject: test.hasSummary,
-				relationships:    test.hasRelationships,
+			fakeCache := common.FakeSummaryCache{
+				SummarizedObject: test.hasSummary,
+				Relationships:    test.hasRelationships,
 			}
 			df := common.DefaultFields{
 				Cache: &fakeCache,
 			}
-			output, err := df.GetTransform()(test.input)
-			require.Equal(t, test.wantOutput, output)
+			raw, isSignal, err := common.GetUnstructured(test.input)
+			if err != nil {
+				require.True(t, test.wantError)
+				return
+			}
+			if isSignal {
+				require.Equal(t, test.input, test.wantOutput)
+				return
+			}
+			output, err := df.TransformCommon(raw)
 			if test.wantError {
 				require.Error(t, err)
 			} else {
+				require.Equal(t, test.wantOutput, output)
 				require.NoError(t, err)
 			}
 		})
 	}
 }
-
-type fakeSummaryCache struct {
-	summarizedObject *summary.SummarizedObject
-	relationships    []summarycache.Relationship
-}
-
-func (f *fakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) {
-	return f.summarizedObject, f.relationships
-}

pkg/resources/virtual/common/testutil.go

@@ -0,0 +1,16 @@
+package common
+
+import (
+	"github.com/rancher/steve/pkg/summarycache"
+	"github.com/rancher/wrangler/v3/pkg/summary"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type FakeSummaryCache struct {
+	SummarizedObject *summary.SummarizedObject
+	Relationships    []summarycache.Relationship
+}
+
+func (f *FakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) {
+	return f.SummarizedObject, f.Relationships
+}

pkg/resources/virtual/common/util.go

@@ -11,7 +11,7 @@ import (
 // GetUnstructured retrieves an unstructured object from the provided input. If this is a signal
 // object (like cache.DeletedFinalStateUnknown), returns true, indicating that this wasn't an
 // unstructured object, but doesn't need to be processed by our transform function
-func getUnstructured(obj any) (*unstructured.Unstructured, bool, error) {
+func GetUnstructured(obj any) (*unstructured.Unstructured, bool, error) {
 	raw, ok := obj.(*unstructured.Unstructured)
 	if !ok {
 		_, isFinalUnknown := obj.(cache.DeletedFinalStateUnknown)

pkg/resources/virtual/events/events.go

@@ -0,0 +1,16 @@
+// Package common provides cache.TransformFunc's for /v1 Event objects
+package events
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// TransformEventObject does special-case handling on event objects
+// 1. (only one so far): replaces the _type field with the contents of the field named "type", if it exists
+func TransformEventObject(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
+	currentTypeValue, ok := obj.Object["type"]
+	if ok {
+		obj.Object["_type"] = currentTypeValue
+	}
+	return obj, nil
+}

pkg/resources/virtual/events/events_test.go

@@ -0,0 +1,93 @@
+package events_test
+
+import (
+	"testing"
+
+	"github.com/rancher/steve/pkg/resources/virtual/events"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func TestTransformEvents(t *testing.T) {
+	tests := []struct {
+		name       string
+		input      any
+		wantOutput any
+		wantError  bool
+	}{
+		{
+			name: "fix event fields",
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":      "gregsFarm",
+						"namespace": "gregsNamespace",
+					},
+					"id":   "eventTest1id",
+					"type": "Gorniplatz",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":      "gregsFarm",
+						"namespace": "gregsNamespace",
+					},
+					"id":    "eventTest1id",
+					"type":  "Gorniplatz",
+					"_type": "Gorniplatz",
+				},
+			},
+		},
+		{
+			name: "don't fix non-default-group event fields",
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "palau.io/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":      "gregsFarm",
+						"namespace": "gregsNamespace",
+					},
+					"id":   "eventTest1id",
+					"type": "Gorniplatz",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "palau.io/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":      "gregsFarm",
+						"namespace": "gregsNamespace",
+					},
+					"id":   "eventTest1id",
+					"type": "Gorniplatz",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var output interface{}
+			var err error
+			raw, ok := test.input.(*unstructured.Unstructured)
+			if ok && raw.GetKind() == "Event" && raw.GetAPIVersion() == "/v1" {
+				output, err = events.TransformEventObject(raw)
+			} else {
+				output = raw
+				err = nil
+			}
+			require.Equal(t, test.wantOutput, output)
+			if test.wantError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

pkg/resources/virtual/virtual.go

@@ -3,7 +3,11 @@
 package virtual
 
 import (
+	"fmt"
+
 	"github.com/rancher/steve/pkg/resources/virtual/common"
+	"github.com/rancher/steve/pkg/resources/virtual/events"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/tools/cache"
 )
@@ -16,13 +20,36 @@ type TransformBuilder struct {
 // NewTransformBuilder returns a TransformBuilder using the given summary cache
 func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder {
 	return &TransformBuilder{
-		&common.DefaultFields{
+		defaultFields: &common.DefaultFields{
 			Cache: cache,
 		},
 	}
 }
 
-// GetTransformFunc retrieves a TransformFunc for a given GVK. Currently only returns a transformFunc for defaultFields
-func (t *TransformBuilder) GetTransformFunc(_ schema.GroupVersionKind) cache.TransformFunc {
-	return t.defaultFields.GetTransform()
+// GetTransformFunc returns the func to transform a raw object into a fixed object, if needed
+func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc {
+	converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0)
+	if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" {
+		converters = append(converters, events.TransformEventObject)
+	}
+	converters = append(converters, t.defaultFields.TransformCommon)
+
+	return func(raw interface{}) (interface{}, error) {
+		obj, isSignal, err := common.GetUnstructured(raw)
+		if isSignal {
+			// isSignal= true overrides any error
+			return raw, err
+		}
+		if err != nil {
+			return nil, fmt.Errorf("GetUnstructured: failed to get underlying object: %w", err)
+		}
+		// Conversions are run in this loop:
+		for _, f := range converters {
+			obj, err = f(obj)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return obj, nil
+	}
 }

pkg/resources/virtual/virtual_test.go

@@ -0,0 +1,202 @@
+package virtual_test
+
+import (
+	"github.com/rancher/steve/pkg/resources/virtual"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"strings"
+	"testing"
+
+	"github.com/rancher/steve/pkg/resources/virtual/common"
+	"github.com/rancher/steve/pkg/summarycache"
+	"github.com/rancher/wrangler/v3/pkg/summary"
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func TestTransformChain(t *testing.T) {
+	tests := []struct {
+		name             string
+		input            any
+		hasSummary       *summary.SummarizedObject
+		hasRelationships []summarycache.Relationship
+		wantOutput       any
+		wantError        bool
+	}{
+		{
+			name: "add summary + relationships + reserved fields",
+			hasSummary: &summary.SummarizedObject{
+				PartialObjectMetadata: v1.PartialObjectMetadata{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "testobj",
+						Namespace: "test-ns",
+					},
+					TypeMeta: v1.TypeMeta{
+						APIVersion: "test.cattle.io/v1",
+						Kind:       "TestResource",
+					},
+				},
+				Summary: summary.Summary{
+					State:         "success",
+					Transitioning: false,
+					Error:         false,
+					Message:       []string{"resource 1 rolled out", "resource 2 rolled out"},
+				},
+			},
+			hasRelationships: []summarycache.Relationship{
+				{
+					ToID:        "1345",
+					ToType:      "SomeType",
+					ToNamespace: "some-ns",
+					FromID:      "78901",
+					FromType:    "TestResource",
+					Rel:         "uses",
+				},
+			},
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+					},
+					"id": "old-id",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+						"state": map[string]interface{}{
+							"name":          "success",
+							"error":         false,
+							"transitioning": false,
+							"message":       "resource 1 rolled out:resource 2 rolled out",
+						},
+						"relationships": []any{
+							map[string]any{
+								"toId":        "1345",
+								"toType":      "SomeType",
+								"toNamespace": "some-ns",
+								"fromId":      "78901",
+								"fromType":    "TestResource",
+								"rel":         "uses",
+							},
+						},
+					},
+					"id":  "test-ns/testobj",
+					"_id": "old-id",
+				},
+			},
+		},
+		{
+			name: "processable event",
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":      "oswaldsFarm",
+						"namespace": "oswaldsNamespace",
+					},
+					"status": map[string]interface{}{
+						"conditions": []interface{}{
+							map[string]interface{}{
+								"status":             "False",
+								"reason":             "Error",
+								"message":            "some error",
+								"lastTransitionTime": "2024-01-01",
+							},
+						},
+					},
+					"id":   "eventTest2id",
+					"type": "Gorniplatz",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":          "oswaldsFarm",
+						"namespace":     "oswaldsNamespace",
+						"relationships": []any(nil),
+					},
+					"status": map[string]interface{}{
+						"conditions": []interface{}{
+							map[string]interface{}{
+								"status":             "False",
+								"reason":             "Error",
+								"transitioning":      false,
+								"error":              true,
+								"message":            "some error",
+								"lastTransitionTime": "2024-01-01",
+								"lastUpdateTime":     "2024-01-01",
+							},
+						},
+					},
+					"id":    "oswaldsNamespace/oswaldsFarm",
+					"_id":   "eventTest2id",
+					"type":  "Gorniplatz",
+					"_type": "Gorniplatz",
+				},
+			},
+		},
+		{
+			name: "don't fix non-default-group event fields",
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "palau.io/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":          "gregsFarm",
+						"namespace":     "gregsNamespace",
+						"relationships": []any(nil),
+					},
+					"id":   "eventTest1id",
+					"type": "Gorniplatz",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "palau.io/v1",
+					"kind":       "Event",
+					"metadata": map[string]interface{}{
+						"name":          "gregsFarm",
+						"namespace":     "gregsNamespace",
+						"relationships": []any(nil),
+					},
+					"id":   "gregsNamespace/gregsFarm",
+					"_id":  "eventTest1id",
+					"type": "Gorniplatz",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeCache := common.FakeSummaryCache{
+				SummarizedObject: test.hasSummary,
+				Relationships:    test.hasRelationships,
+			}
+			tb := virtual.NewTransformBuilder(&fakeCache)
+			raw, isSignal, err := common.GetUnstructured(test.input)
+			require.False(t, isSignal)
+			require.Nil(t, err)
+			apiVersion := raw.GetAPIVersion()
+			parts := strings.Split(apiVersion, "/")
+			gvk := schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: raw.GetKind()}
+			output, err := tb.GetTransformFunc(gvk)(test.input)
+			require.Equal(t, test.wantOutput, output)
+			if test.wantError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

pkg/stores/sqlproxy/proxy_store.go

@@ -61,12 +61,47 @@ var (
 	paramScheme               = runtime.NewScheme()
 	paramCodec                = runtime.NewParameterCodec(paramScheme)
 	typeSpecificIndexedFields = map[string][][]string{
-		"_v1_Namespace": {{`metadata`, `labels[field.cattle.io/projectId]`}},
-		"_v1_Node":      {{`status`, `nodeInfo`, `kubeletVersion`}, {`status`, `nodeInfo`, `operatingSystem`}},
-		"_v1_Pod":       {{`spec`, `containers`, `image`}, {`spec`, `nodeName`}},
-		"_v1_ConfigMap": {{`metadata`, `labels[harvesterhci.io/cloud-init-template]`}},
-
-		"management.cattle.io_v3_Node": {{`status`, `nodeName`}},
+		gvkKey("", "v1", "Event"): {
+			{"_type"},
+			{"involvedObject", "kind"},
+			{"message"},
+			{"reason"},
+		},
+		gvkKey("", "v1", "Namespace"): {
+			{"metadata", "labels[field.cattle.io/projectId]"}},
+		gvkKey("", "v1", "Node"): {
+			{"status", "nodeInfo", "kubeletVersion"},
+			{"status", "nodeInfo", "operatingSystem"}},
+		gvkKey("", "v1", "Pod"): {
+			{"spec", "containers", "image"},
+			{"spec", "nodeName"}},
+		gvkKey("", "v1", "ConfigMap"): {
+			{"metadata", "labels[harvesterhci.io/cloud-init-template]"}},
+		gvkKey("catalog.cattle.io", "v1", "ClusterRepo"): {
+			{"metadata", "annotations[clusterrepo.cattle.io/hidden]"},
+			{"spec", "gitBranch"},
+			{"spec", "gitRepo"},
+		},
+		gvkKey("catalog.cattle.io", "v1", "Operation"): {
+			{"status", "action"},
+			{"status", "namespace"},
+			{"status", "releaseName"},
+		},
+		gvkKey("cluster.x-k8s.io", "v1beta1", "Machine"): {
+			{"spec", "clusterName"}},
+		gvkKey("management.cattle.io", "v3", "Node"): {
+			{"status", "nodeName"}},
+		gvkKey("management.cattle.io", "v3", "NodePool"): {
+			{"spec", "clusterName"}},
+		gvkKey("management.cattle.io", "v3", "NodeTemplate"): {
+			{"spec", "clusterName"}},
+		gvkKey("provisioning.cattle.io", "v1", "Cluster"): {
+			{"metadata", "labels[provider.cattle.io]"},
+			{"status", "provider"},
+			{"status", "allocatable", "cpu"},
+			{"status", "allocatable", "memory"},
+			{"status", "allocatable", "pods"},
+		},
 	}
 	commonIndexFields = [][]string{
 		{`id`},
@@ -241,15 +276,15 @@ func (s *Store) initializeNamespaceCache() error {
 func getFieldForGVK(gvk schema.GroupVersionKind) [][]string {
 	fields := [][]string{}
 	fields = append(fields, commonIndexFields...)
-	typeFields := typeSpecificIndexedFields[keyFromGVK(gvk)]
+	typeFields := typeSpecificIndexedFields[gvkKey(gvk.Group, gvk.Version, gvk.Kind)]
 	if typeFields != nil {
 		fields = append(fields, typeFields...)
 	}
 	return fields
 }
 
-func keyFromGVK(gvk schema.GroupVersionKind) string {
-	return gvk.Group + "_" + gvk.Version + "_" + gvk.Kind
+func gvkKey(group, version, kind string) string {
+	return group + "_" + version + "_" + kind
 }
 
 // getFieldsFromSchema converts object field names from types.APISchema's format into lasso's