[v2.9] Virtual Resource filters (#288)

Adds logic which adds virtual fields resources. This allows these fields
to be sorted/filtered on when the SQL cache is enabled. Id and
metadata.state.name were added as the first two fields.

Co-authored-by: Michael Bolot <michael.bolot@suse.com>

README.md

@@ -124,6 +124,17 @@ item is included in the list.
 /v1/{type}?filter=spec.containers.image=alpine
 ```
 
+**If SQLite caching is enabled** (`server.Options.SQLCache=true`),
+filtering is only supported for a subset of attributes:
+- `id`, `metadata.name`, `metadata.namespace`, `metadata.state.name`, and `metadata.timestamp` for any resource kind
+- a short list of hardcoded attributes for a selection of specific types listed
+in [typeSpecificIndexFields](https://github.com/rancher/steve/blob/main/pkg/stores/sqlproxy/proxy_store.go#L52-L58)
+- the special string `metadata.fields[N]`, with N starting at 0, for all columns
+displayed by `kubectl get $TYPE`. For example `secrets` have `"metadata.fields[0]"`,
+`"metadata.fields[1]"` , `"metadata.fields[2]"`, and `"metadata.fields[3]"` respectively
+corresponding to `"name"`, `"type"`, `"data"`, and `"age"`. For CRDs, these come from
+[Additional printer columns](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#additional-printer-columns)
+
 #### `projectsornamespaces`
 
 Resources can also be filtered by the Rancher projects their namespaces belong

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146
 	github.com/rancher/dynamiclistener v0.6.0-rc2
 	github.com/rancher/kubernetes-provider-detector v0.1.5
-	github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1
+	github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589
 	github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9
 	github.com/rancher/remotedialer v0.3.2
 	github.com/rancher/wrangler/v3 v3.0.0
@@ -80,7 +80,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -109,7 +109,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/component-base v0.30.1 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.49.3 // indirect
 	modernc.org/mathutil v1.6.0 // indirect

go.sum

@@ -184,16 +184,16 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146 h1:6I4Z7PAGmned9+EYxbMS7kvajId3r8+ZwAR5wB7X3kg=
 github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146/go.mod h1:ZNk+LcRGwQYHqgbsJijRrI49KFbX31/QzoUBq4rAeV0=
 github.com/rancher/dynamiclistener v0.6.0-rc2 h1:ASh61tOKTa2OJyKMc9stcmv7W6Xn/rwA8Me0yEIUe7s=
 github.com/rancher/dynamiclistener v0.6.0-rc2/go.mod h1:7VNEQhAwzbYJ08S1MYb6B4vili6K7CcrG4cNZXq1j+s=
 github.com/rancher/kubernetes-provider-detector v0.1.5 h1:hWRAsWuJOemzGjz/XrbTlM7QmfO4OedvFE3QwXiH60I=
 github.com/rancher/kubernetes-provider-detector v0.1.5/go.mod h1:ypuJS7kP7rUiAn330xG46mj+Nhvym05GM8NqMVekpH0=
-github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1 h1:vv1jDlYbd4KhGbPNxmjs8CYgEHUrQm2bMtmULfXJ6iw=
+github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589 h1:c3IIzpVo5Jhd1T7Dih/kbEueWC21j4xTZ5EQO9rhTUg=
-github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1/go.mod h1:A/y3BLQkxZXYD60MNDRwAG9WGxXfvd6Z6gWR/a8wPw8=
+github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589/go.mod h1:+GJaJqWpk1MZYCDrMC+4Jee0M/aScoru5T8V2A//rgg=
 github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9 h1:AlRMRs5mHJcdiK83KKJyFVeybPMZ7dOUzC0l3k9aUa8=
 github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9/go.mod h1:dyjfXBsNiroPWOdUZe7diUOUSLf6HQ/r2kEpwH/8zas=
 github.com/rancher/remotedialer v0.3.2 h1:kstZbRwPS5gPWpGg8VjEHT2poHtArs+Fc317YM8JCzU=
@@ -252,6 +252,8 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -412,8 +414,8 @@ k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C
 k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3 h1:SbdLaI6mM6ffDSJCadEaD4IkuPzepLDGlkd2xV0t1uA=
 k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
+k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk=
 modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
 modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA=

pkg/resources/common/formatter.go

@@ -23,18 +23,19 @@ import (
 func DefaultTemplate(clientGetter proxy.ClientGetter,
 	summaryCache *summarycache.SummaryCache,
 	asl accesscontrol.AccessSetLookup,
-	namespaceCache corecontrollers.NamespaceCache) schema.Template {
+	namespaceCache corecontrollers.NamespaceCache,
+	sqlCache bool) schema.Template {
 	return schema.Template{
 		Store:     metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)),
-		Formatter: formatter(summaryCache),
+		Formatter: formatter(summaryCache, sqlCache),
 	}
 }
 
 // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
-func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache) schema.Template {
+func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache, sqlCache bool) schema.Template {
 	return schema.Template{
 		Store:     store,
-		Formatter: formatter(summaryCache),
+		Formatter: formatter(summaryCache, sqlCache),
 	}
 }
 
@@ -71,7 +72,7 @@ func selfLink(gvr schema2.GroupVersionResource, meta metav1.Object) (prefix stri
 	return buf.String()
 }
 
-func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
+func formatter(summarycache *summarycache.SummaryCache, sqlCache bool) types.Formatter {
 	return func(request *types.APIRequest, resource *types.RawResource) {
 		if resource.Schema == nil {
 			return
@@ -104,6 +105,8 @@ func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
 		}
 
 		if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok {
+			if !sqlCache {
+				// with the sql cache, these were already added by the indexer
 				s, rel := summarycache.SummaryAndRelationship(unstr)
 				data.PutValue(unstr.Object, map[string]interface{}{
 					"name":          s.State,
@@ -115,6 +118,7 @@ func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
 
 				summary.NormalizeConditions(unstr)
 
+			}
 			includeFields(request, unstr)
 			excludeFields(request, unstr)
 			excludeValues(request, unstr)

pkg/resources/schema.go

@@ -49,7 +49,7 @@ func DefaultSchemaTemplates(cf *client.Factory,
 	discovery discovery.DiscoveryInterface,
 	namespaceCache corecontrollers.NamespaceCache) []schema.Template {
 	return []schema.Template{
-		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache),
+		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, false),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",
@@ -79,7 +79,7 @@ func DefaultSchemaTemplatesForStore(store types.Store,
 	discovery discovery.DiscoveryInterface) []schema.Template {
 
 	return []schema.Template{
-		common.DefaultTemplateForStore(store, summaryCache),
+		common.DefaultTemplateForStore(store, summaryCache, true),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",

pkg/resources/virtual/common/common.go

@@ -0,0 +1,116 @@
+// Package common provides cache.TransformFunc's which are common to all types
+package common
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/rancher/steve/pkg/summarycache"
+	"github.com/rancher/wrangler/v3/pkg/data"
+	wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+// SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache
+// struct from pkg/summarycache
+type SummaryCache interface {
+	SummaryAndRelationship(runtime.Object) (*wranglerSummary.SummarizedObject, []summarycache.Relationship)
+}
+
+// DefaultFields produces a VirtualTransformFunc through GetTransform() that applies to all k8s types
+type DefaultFields struct {
+	Cache SummaryCache
+}
+
+// GetTransform produces the default transformation func
+func (d *DefaultFields) GetTransform() cache.TransformFunc {
+	return d.transform
+}
+
+// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary
+func (d *DefaultFields) transform(obj any) (any, error) {
+	raw, isSignal, err := getUnstructured(obj)
+	if isSignal {
+		return obj, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	raw = addIDField(raw)
+	raw, err = addSummaryFields(raw, d.Cache)
+	if err != nil {
+		return nil, fmt.Errorf("unable to add summary fields: %w", err)
+	}
+	return raw, nil
+}
+
+// addSummaryFields adds the virtual fields for object state.
+func addSummaryFields(raw *unstructured.Unstructured, cache SummaryCache) (*unstructured.Unstructured, error) {
+	s, relationships := cache.SummaryAndRelationship(raw)
+	if s != nil {
+		data.PutValue(raw.Object, map[string]interface{}{
+			"name":          s.State,
+			"error":         s.Error,
+			"transitioning": s.Transitioning,
+			"message":       strings.Join(s.Message, ":"),
+		}, "metadata", "state")
+
+	}
+	var rels []any
+	for _, relationship := range relationships {
+		rel, err := toMap(relationship)
+		if err != nil {
+			return nil, fmt.Errorf("unable to convert relationship to map: %w", err)
+		}
+		rels = append(rels, rel)
+	}
+	data.PutValue(raw.Object, rels, "metadata", "relationships")
+
+	normalizeConditions(raw)
+	return raw, nil
+}
+
+// addIDField adds the ID field based on namespace/name, and moves the current id field to _id if present
+func addIDField(raw *unstructured.Unstructured) *unstructured.Unstructured {
+	objectID := raw.GetName()
+	namespace := raw.GetNamespace()
+	if namespace != "" {
+		objectID = fmt.Sprintf("%s/%s", namespace, objectID)
+	}
+	currentIDValue, ok := raw.Object["id"]
+	if ok {
+		raw.Object["_id"] = currentIDValue
+	}
+	raw.Object["id"] = objectID
+	return raw
+}
+
+func normalizeConditions(raw *unstructured.Unstructured) {
+	var (
+		obj           data.Object
+		newConditions []any
+	)
+
+	obj = raw.Object
+	for _, condition := range obj.Slice("status", "conditions") {
+		var summary wranglerSummary.Summary
+		for _, summarizer := range wranglerSummary.ConditionSummarizers {
+			summary = summarizer(obj, []wranglerSummary.Condition{{Object: condition}}, summary)
+		}
+		condition.Set("error", summary.Error)
+		condition.Set("transitioning", summary.Transitioning)
+
+		if condition.String("lastUpdateTime") == "" {
+			condition.Set("lastUpdateTime", condition.String("lastTransitionTime"))
+		}
+		// needs to be reconverted back to a map[string]any or we can have encoding problems with unregistered types
+		var mapCondition map[string]any = condition
+		newConditions = append(newConditions, mapCondition)
+	}
+
+	if len(newConditions) > 0 {
+		obj.SetNested(newConditions, "status", "conditions")
+	}
+}

pkg/resources/virtual/common/common_test.go

@@ -0,0 +1,188 @@
+package common_test
+
+import (
+	"testing"
+
+	"github.com/rancher/steve/pkg/resources/virtual/common"
+	"github.com/rancher/steve/pkg/summarycache"
+	"github.com/rancher/wrangler/v3/pkg/summary"
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+func TestTransform(t *testing.T) {
+	tests := []struct {
+		name             string
+		input            any
+		hasSummary       *summary.SummarizedObject
+		hasRelationships []summarycache.Relationship
+		wantOutput       any
+		wantError        bool
+	}{
+		{
+			name: "signal error",
+			input: cache.DeletedFinalStateUnknown{
+				Key: "some-ns/some-name",
+			},
+			wantOutput: cache.DeletedFinalStateUnknown{
+				Key: "some-ns/some-name",
+			},
+			wantError: false,
+		},
+		{
+			name: "not unstructured",
+			input: map[string]any{
+				"somekey": "someval",
+			},
+			wantError: true,
+		},
+		{
+			name: "add summary + relationships + reserved fields",
+			hasSummary: &summary.SummarizedObject{
+				PartialObjectMetadata: v1.PartialObjectMetadata{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "testobj",
+						Namespace: "test-ns",
+					},
+					TypeMeta: v1.TypeMeta{
+						APIVersion: "test.cattle.io/v1",
+						Kind:       "TestResource",
+					},
+				},
+				Summary: summary.Summary{
+					State:         "success",
+					Transitioning: false,
+					Error:         false,
+					Message:       []string{"resource 1 rolled out", "resource 2 rolled out"},
+				},
+			},
+			hasRelationships: []summarycache.Relationship{
+				{
+					ToID:        "1345",
+					ToType:      "SomeType",
+					ToNamespace: "some-ns",
+					FromID:      "78901",
+					FromType:    "TestResource",
+					Rel:         "uses",
+				},
+			},
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+					},
+					"id": "old-id",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+						"state": map[string]interface{}{
+							"name":          "success",
+							"error":         false,
+							"transitioning": false,
+							"message":       "resource 1 rolled out:resource 2 rolled out",
+						},
+						"relationships": []any{
+							map[string]any{
+								"toId":        "1345",
+								"toType":      "SomeType",
+								"toNamespace": "some-ns",
+								"fromId":      "78901",
+								"fromType":    "TestResource",
+								"rel":         "uses",
+							},
+						},
+					},
+					"id":  "test-ns/testobj",
+					"_id": "old-id",
+				},
+			},
+		},
+		{
+			name: "add conditions + reserved fields",
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+					},
+					"status": map[string]interface{}{
+						"conditions": []interface{}{
+							map[string]interface{}{
+								"status":             "False",
+								"reason":             "Error",
+								"message":            "some error",
+								"lastTransitionTime": "2024-01-01",
+							},
+						},
+					},
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":          "testobj",
+						"namespace":     "test-ns",
+						"relationships": []any(nil),
+					},
+					"status": map[string]interface{}{
+						"conditions": []interface{}{
+							map[string]interface{}{
+								"status":             "False",
+								"reason":             "Error",
+								"transitioning":      false,
+								"error":              true,
+								"message":            "some error",
+								"lastTransitionTime": "2024-01-01",
+								"lastUpdateTime":     "2024-01-01",
+							},
+						},
+					},
+					"id": "test-ns/testobj",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeCache := fakeSummaryCache{
+				summarizedObject: test.hasSummary,
+				relationships:    test.hasRelationships,
+			}
+			df := common.DefaultFields{
+				Cache: &fakeCache,
+			}
+			output, err := df.GetTransform()(test.input)
+			require.Equal(t, test.wantOutput, output)
+			if test.wantError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+type fakeSummaryCache struct {
+	summarizedObject *summary.SummarizedObject
+	relationships    []summarycache.Relationship
+}
+
+func (f *fakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) {
+	return f.summarizedObject, f.relationships
+}

pkg/resources/virtual/common/util.go

@@ -0,0 +1,40 @@
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/tools/cache"
+)
+
+// GetUnstructured retrieves an unstructured object from the provided input. If this is a signal
+// object (like cache.DeletedFinalStateUnknown), returns true, indicating that this wasn't an
+// unstructured object, but doesn't need to be processed by our transform function
+func getUnstructured(obj any) (*unstructured.Unstructured, bool, error) {
+	raw, ok := obj.(*unstructured.Unstructured)
+	if !ok {
+		_, isFinalUnknown := obj.(cache.DeletedFinalStateUnknown)
+		if isFinalUnknown {
+			// As documented in the TransformFunc interface
+			return nil, true, nil
+		}
+		return nil, false, fmt.Errorf("object was of type %T, not unstructured", raw)
+	}
+	return raw, false, nil
+}
+
+// toMap converts an object to a map[string]any which can be stored/retrieved from the cache. Currently
+// uses json encoding to take advantage of tag names
+func toMap(obj any) (map[string]any, error) {
+	bytes, err := json.Marshal(obj)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal object: %w", err)
+	}
+	var retObj map[string]any
+	err = json.Unmarshal(bytes, &retObj)
+	if err != nil {
+		return nil, fmt.Errorf("unable to unmarshal object: %w", err)
+	}
+	return retObj, nil
+}

pkg/resources/virtual/virtual.go

@@ -0,0 +1,28 @@
+// Package virtual provides functions/resources to define virtual fields (fields which don't exist in k8s
+// but should be visible in the API) on resources
+package virtual
+
+import (
+	"github.com/rancher/steve/pkg/resources/virtual/common"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/tools/cache"
+)
+
+// TransformBuilder builds transform functions for specified GVKs through GetTransformFunc
+type TransformBuilder struct {
+	defaultFields *common.DefaultFields
+}
+
+// NewTransformBuilder returns a TransformBuilder using the given summary cache
+func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder {
+	return &TransformBuilder{
+		&common.DefaultFields{
+			Cache: cache,
+		},
+	}
+}
+
+// GetTransformFunc retrieves a TransformFunc for a given GVK. Currently only returns a transformFunc for defaultFields
+func (t *TransformBuilder) GetTransformFunc(_ schema.GroupVersionKind) cache.TransformFunc {
+	return t.defaultFields.GetTransform()
+}

pkg/server/server.go

@@ -163,7 +163,7 @@ func setup(ctx context.Context, server *Server) error {
 
 	var onSchemasHandler schemacontroller.SchemasHandlerFunc
 	if server.SQLCache {
-		s, err := sqlproxy.NewProxyStore(cols, cf, summaryCache, nil)
+		s, err := sqlproxy.NewProxyStore(cols, cf, summaryCache, summaryCache, nil)
 		if err != nil {
 			panic(err)
 		}

pkg/stores/partition/store.go

@@ -111,7 +111,7 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return ToAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // ByID looks up a single object by its ID.
@@ -125,7 +125,7 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return ToAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 func (s *Store) listPartition(ctx context.Context, apiOp *types.APIRequest, schema *types.APISchema, partition Partition,
@@ -227,7 +227,7 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
 
 	for _, item := range list {
 		item := item.DeepCopy()
-		result.Objects = append(result.Objects, ToAPI(schema, item, nil))
+		result.Objects = append(result.Objects, ToAPI(schema, item, nil, types.ReservedFields))
 	}
 
 	result.Pages = pages
@@ -267,7 +267,7 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return ToAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // Update updates a single object in the store.
@@ -281,7 +281,7 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return ToAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // Watch returns a channel of events for a list or resource.
@@ -327,13 +327,13 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types
 	return response, nil
 }
 
-func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning) types.APIObject {
+func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning, reservedFields map[string]bool) types.APIObject {
 	if obj == nil || reflect.ValueOf(obj).IsNil() {
 		return types.APIObject{}
 	}
 
 	if unstr, ok := obj.(*unstructured.Unstructured); ok {
-		obj = moveToUnderscore(unstr)
+		obj = moveToUnderscore(unstr, reservedFields)
 	}
 
 	apiObject := types.APIObject{
@@ -357,12 +357,12 @@ func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning
 	return apiObject
 }
 
-func moveToUnderscore(obj *unstructured.Unstructured) *unstructured.Unstructured {
+func moveToUnderscore(obj *unstructured.Unstructured, reservedFields map[string]bool) *unstructured.Unstructured {
 	if obj == nil {
 		return nil
 	}
 
-	for k := range types.ReservedFields {
+	for k := range reservedFields {
 		v, ok := obj.Object[k]
 		if ok {
 			delete(obj.Object, k)
@@ -394,7 +394,7 @@ func ToAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Ev
 		return apiEvent
 	}
 
-	apiEvent.Object = ToAPI(schema, event.Object, nil)
+	apiEvent.Object = ToAPI(schema, event.Object, nil, types.ReservedFields)
 
 	m, err := meta.Accessor(event.Object)
 	if err != nil {

pkg/stores/sqlpartition/store.go

@@ -26,6 +26,7 @@ type SchemaColumnSetter interface {
 type Store struct {
 	Partitioner       Partitioner
 	asl               accesscontrol.AccessSetLookup
+	sqlReservedFields map[string]bool
 }
 
 // NewStore creates a types.proxyStore implementation with a partitioner
@@ -36,6 +37,14 @@ func NewStore(store UnstructuredStore, asl accesscontrol.AccessSetLookup) *Store
 		},
 		asl: asl,
 	}
+	sqlReservedFields := map[string]bool{}
+	for key, value := range types.ReservedFields {
+		if key == "id" {
+			continue
+		}
+		sqlReservedFields[key] = value
+	}
+	s.sqlReservedFields = sqlReservedFields
 
 	return s
 }
@@ -48,7 +57,7 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return partition.ToAPI(schema, obj, warnings), nil
+	return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // ByID looks up a single object by its ID.
@@ -59,7 +68,7 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return partition.ToAPI(schema, obj, warnings), nil
+	return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // List returns a list of objects across all applicable partitions.
@@ -85,7 +94,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
 
 	for _, item := range list {
 		item := item.DeepCopy()
-		result.Objects = append(result.Objects, partition.ToAPI(schema, item, nil))
+		// the sql cache automatically adds the ID through a transformFunc. Because of this, we have a different set of reserved fields for the SQL cache
+		result.Objects = append(result.Objects, partition.ToAPI(schema, item, nil, s.sqlReservedFields))
 	}
 
 	result.Revision = ""
@@ -101,7 +111,7 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return partition.ToAPI(schema, obj, warnings), nil
+	return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // Update updates a single object in the store.
@@ -112,7 +122,7 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return partition.ToAPI(schema, obj, warnings), nil
+	return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil
 }
 
 // Watch returns a channel of events for a list or resource.

pkg/stores/sqlproxy/proxy_mocks_test.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/rancher/steve/pkg/stores/sqlproxy (interfaces: Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier)
+// Source: github.com/rancher/steve/pkg/stores/sqlproxy (interfaces: Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier,TransformBuilder)
 
 // Package sqlproxy is a generated GoMock package.
 package sqlproxy
@@ -19,6 +19,7 @@ import (
 	dynamic "k8s.io/client-go/dynamic"
 	kubernetes "k8s.io/client-go/kubernetes"
 	rest "k8s.io/client-go/rest"
+	cache "k8s.io/client-go/tools/cache"
 )
 
 // MockCache is a mock of Cache interface.
@@ -257,18 +258,18 @@ func (m *MockCacheFactory) EXPECT() *MockCacheFactoryMockRecorder {
 }
 
 // CacheFor mocks base method.
-func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 dynamic.ResourceInterface, arg2 schema.GroupVersionKind, arg3 bool) (factory.Cache, error) {
+func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 cache.TransformFunc, arg2 dynamic.ResourceInterface, arg3 schema.GroupVersionKind, arg4 bool) (factory.Cache, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3)
+	ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(factory.Cache)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // CacheFor indicates an expected call of CacheFor.
-func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3, arg4)
 }
 
 // Reset mocks base method.
@@ -358,3 +359,40 @@ func (mr *MockRelationshipNotifierMockRecorder) OnInboundRelationshipChange(arg0
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnInboundRelationshipChange", reflect.TypeOf((*MockRelationshipNotifier)(nil).OnInboundRelationshipChange), arg0, arg1, arg2)
 }
+
+// MockTransformBuilder is a mock of TransformBuilder interface.
+type MockTransformBuilder struct {
+	ctrl     *gomock.Controller
+	recorder *MockTransformBuilderMockRecorder
+}
+
+// MockTransformBuilderMockRecorder is the mock recorder for MockTransformBuilder.
+type MockTransformBuilderMockRecorder struct {
+	mock *MockTransformBuilder
+}
+
+// NewMockTransformBuilder creates a new mock instance.
+func NewMockTransformBuilder(ctrl *gomock.Controller) *MockTransformBuilder {
+	mock := &MockTransformBuilder{ctrl: ctrl}
+	mock.recorder = &MockTransformBuilderMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockTransformBuilder) EXPECT() *MockTransformBuilderMockRecorder {
+	return m.recorder
+}
+
+// GetTransformFunc mocks base method.
+func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind) cache.TransformFunc {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetTransformFunc", arg0)
+	ret0, _ := ret[0].(cache.TransformFunc)
+	return ret0
+}
+
+// GetTransformFunc indicates an expected call of GetTransformFunc.
+func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0)
+}

pkg/stores/sqlproxy/proxy_store.go

@@ -15,6 +15,15 @@ import (
 	"sync"
 
 	"github.com/pkg/errors"
+
+	"github.com/rancher/steve/pkg/attributes"
+	"github.com/rancher/steve/pkg/resources/common"
+	"github.com/rancher/steve/pkg/resources/virtual"
+	virtualCommon "github.com/rancher/steve/pkg/resources/virtual/common"
+	metricsStore "github.com/rancher/steve/pkg/stores/metrics"
+	"github.com/rancher/steve/pkg/stores/sqlpartition/listprocessor"
+	"github.com/rancher/steve/pkg/stores/sqlproxy/tablelistconvert"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -39,11 +48,7 @@ import (
 	"github.com/rancher/wrangler/v3/pkg/schemas/validation"
 	"github.com/rancher/wrangler/v3/pkg/summary"
 
-	"github.com/rancher/steve/pkg/attributes"
+	"k8s.io/client-go/tools/cache"
-	"github.com/rancher/steve/pkg/resources/common"
-	metricsStore "github.com/rancher/steve/pkg/stores/metrics"
-	"github.com/rancher/steve/pkg/stores/sqlpartition/listprocessor"
-	"github.com/rancher/steve/pkg/stores/sqlproxy/tablelistconvert"
 )
 
 const (
@@ -63,6 +68,10 @@ var (
 
 		"management.cattle.io_v3_Node": {{`status`, `nodeName`}},
 	}
+	commonIndexFields = [][]string{
+		{`id`},
+		{`metadata`, `state`, `name`},
+	}
 	baseNSSchema = types.APISchema{
 		Schema: &schemas.Schema{
 			Attributes: map[string]interface{}{
@@ -124,6 +133,10 @@ type RelationshipNotifier interface {
 	OnInboundRelationshipChange(ctx context.Context, schema *types.APISchema, namespace string) <-chan *summary.Relationship
 }
 
+type TransformBuilder interface {
+	GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc
+}
+
 type Store struct {
 	clientGetter     ClientGetter
 	notifier         RelationshipNotifier
@@ -132,21 +145,23 @@ type Store struct {
 	namespaceCache   Cache
 	lock             sync.Mutex
 	columnSetter     SchemaColumnSetter
+	transformBuilder TransformBuilder
 }
 
 type CacheFactoryInitializer func() (CacheFactory, error)
 
 type CacheFactory interface {
-	CacheFor(fields [][]string, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool) (factory.Cache, error)
+	CacheFor(fields [][]string, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool) (factory.Cache, error)
 	Reset() error
 }
 
 // NewProxyStore returns a Store implemented directly on top of kubernetes.
-func NewProxyStore(c SchemaColumnSetter, clientGetter ClientGetter, notifier RelationshipNotifier, factory CacheFactory) (*Store, error) {
+func NewProxyStore(c SchemaColumnSetter, clientGetter ClientGetter, notifier RelationshipNotifier, scache virtualCommon.SummaryCache, factory CacheFactory) (*Store, error) {
 	store := &Store{
 		clientGetter:     clientGetter,
 		notifier:         notifier,
 		columnSetter:     c,
+		transformBuilder: virtual.NewTransformBuilder(scache),
 	}
 
 	if factory == nil {
@@ -203,14 +218,18 @@ func (s *Store) initializeNamespaceCache() error {
 		return err
 	}
 
+	gvk := attributes.GVK(&nsSchema)
 	// get fields from schema's columns
 	fields := getFieldsFromSchema(&nsSchema)
 
 	// get any type-specific fields that steve is interested in
-	fields = append(fields, getFieldForGVK(attributes.GVK(&nsSchema))...)
+	fields = append(fields, getFieldForGVK(gvk)...)
+
+	// get the type-specifc transform func
+	transformFunc := s.transformBuilder.GetTransformFunc(gvk)
 
 	// get the ns informer
-	nsInformer, err := s.cacheFactory.CacheFor(fields, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false)
+	nsInformer, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false)
 	if err != nil {
 		return err
 	}
@@ -220,7 +239,13 @@ func (s *Store) initializeNamespaceCache() error {
 }
 
 func getFieldForGVK(gvk schema.GroupVersionKind) [][]string {
-	return typeSpecificIndexedFields[keyFromGVK(gvk)]
+	fields := [][]string{}
+	fields = append(fields, commonIndexFields...)
+	typeFields := typeSpecificIndexedFields[keyFromGVK(gvk)]
+	if typeFields != nil {
+		fields = append(fields, typeFields...)
+	}
+	return fields
 }
 
 func keyFromGVK(gvk schema.GroupVersionKind) string {
@@ -639,10 +664,12 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, schema *types.APISchem
 	if err != nil {
 		return nil, 0, "", err
 	}
+	gvk := attributes.GVK(schema)
 	fields := getFieldsFromSchema(schema)
-	fields = append(fields, getFieldForGVK(attributes.GVK(schema))...)
+	fields = append(fields, getFieldForGVK(gvk)...)
+	transformFunc := s.transformBuilder.GetTransformFunc(gvk)
 
-	inf, err := s.cacheFactory.CacheFor(fields, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema))
+	inf, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema))
 	if err != nil {
 		return nil, 0, "", err
 	}

pkg/stores/sqlproxy/proxy_store_test.go

@@ -41,7 +41,7 @@ import (
 	clientgotesting "k8s.io/client-go/testing"
 )
 
-//go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./proxy_mocks_test.go github.com/rancher/steve/pkg/stores/sqlproxy Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier
+//go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./proxy_mocks_test.go github.com/rancher/steve/pkg/stores/sqlproxy Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier,TransformBuilder
 //go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./sql_informer_mocks_test.go github.com/rancher/lasso/pkg/cache/sql/informer ByOptionsLister
 //go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./dynamic_mocks_test.go k8s.io/client-go/dynamic ResourceInterface
 
@@ -82,9 +82,9 @@ func TestNewProxyStore(t *testing.T) {
 			nsSchema := baseNSSchema
 			scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
-			cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(c, nil)
+			cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(c, nil)
 
-			s, err := NewProxyStore(scc, cg, rn, cf)
+			s, err := NewProxyStore(scc, cg, rn, nil, cf)
 			assert.Nil(t, err)
 			assert.Equal(t, scc, s.columnSetter)
 			assert.Equal(t, cg, s.clientGetter)
@@ -105,7 +105,7 @@ func TestNewProxyStore(t *testing.T) {
 			nsSchema := baseNSSchema
 			scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(fmt.Errorf("error"))
 
-			s, err := NewProxyStore(scc, cg, rn, cf)
+			s, err := NewProxyStore(scc, cg, rn, nil, cf)
 			assert.Nil(t, err)
 			assert.Equal(t, scc, s.columnSetter)
 			assert.Equal(t, cg, s.clientGetter)
@@ -127,7 +127,7 @@ func TestNewProxyStore(t *testing.T) {
 			scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(nil, fmt.Errorf("error"))
 
-			s, err := NewProxyStore(scc, cg, rn, cf)
+			s, err := NewProxyStore(scc, cg, rn, nil, cf)
 			assert.Nil(t, err)
 			assert.Equal(t, scc, s.columnSetter)
 			assert.Equal(t, cg, s.clientGetter)
@@ -149,9 +149,9 @@ func TestNewProxyStore(t *testing.T) {
 			nsSchema := baseNSSchema
 			scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
-			cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
+			cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
 
-			s, err := NewProxyStore(scc, cg, rn, cf)
+			s, err := NewProxyStore(scc, cg, rn, nil, cf)
 			assert.Nil(t, err)
 			assert.Equal(t, scc, s.columnSetter)
 			assert.Equal(t, cg, s.clientGetter)
@@ -181,6 +181,7 @@ func TestListByPartitions(t *testing.T) {
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			ri := NewMockResourceInterface(gomock.NewController(t))
 			bloi := NewMockByOptionsLister(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 			inf := &informer.Informer{
 				ByOptionsLister: bloi,
 			}
@@ -191,6 +192,7 @@ func TestListByPartitions(t *testing.T) {
 				namespaceCache:   nsi,
 				clientGetter:     cg,
 				cacheFactory:     cf,
+				transformBuilder: tb,
 			}
 			var partitions []partition.Partition
 			req := &types.APIRequest{
@@ -238,7 +240,8 @@ func TestListByPartitions(t *testing.T) {
 			assert.Nil(t, err)
 			cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
 			// This tests that fields are being extracted from schema columns and the type specific fields map
-			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
+			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
 			list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
 			assert.Nil(t, err)
@@ -253,11 +256,13 @@ func TestListByPartitions(t *testing.T) {
 			nsi := NewMockCache(gomock.NewController(t))
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
 				clientGetter:     cg,
 				cacheFactory:     cf,
+				transformBuilder: tb,
 			}
 			var partitions []partition.Partition
 			req := &types.APIRequest{
@@ -317,11 +322,13 @@ func TestListByPartitions(t *testing.T) {
 			nsi := NewMockCache(gomock.NewController(t))
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
 				clientGetter:     cg,
 				cacheFactory:     cf,
+				transformBuilder: tb,
 			}
 			var partitions []partition.Partition
 			req := &types.APIRequest{
@@ -380,11 +387,13 @@ func TestListByPartitions(t *testing.T) {
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			ri := NewMockResourceInterface(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
 				clientGetter:     cg,
 				cacheFactory:     cf,
+				transformBuilder: tb,
 			}
 			var partitions []partition.Partition
 			req := &types.APIRequest{
@@ -432,7 +441,8 @@ func TestListByPartitions(t *testing.T) {
 			assert.Nil(t, err)
 			cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
 			// This tests that fields are being extracted from schema columns and the type specific fields map
-			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(factory.Cache{}, fmt.Errorf("error"))
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(factory.Cache{}, fmt.Errorf("error"))
 
 			_, _, _, err = s.ListByPartitions(req, schema, partitions)
 			assert.NotNil(t, err)
@@ -447,6 +457,7 @@ func TestListByPartitions(t *testing.T) {
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			ri := NewMockResourceInterface(gomock.NewController(t))
 			bloi := NewMockByOptionsLister(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 			inf := &informer.Informer{
 				ByOptionsLister: bloi,
 			}
@@ -457,6 +468,7 @@ func TestListByPartitions(t *testing.T) {
 				namespaceCache:   nsi,
 				clientGetter:     cg,
 				cacheFactory:     cf,
+				transformBuilder: tb,
 			}
 			var partitions []partition.Partition
 			req := &types.APIRequest{
@@ -504,8 +516,9 @@ func TestListByPartitions(t *testing.T) {
 			assert.Nil(t, err)
 			cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
 			// This tests that fields are being extracted from schema columns and the type specific fields map
-			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
+			cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
 			bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error"))
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 
 			_, _, _, err = s.ListByPartitions(req, schema, partitions)
 			assert.NotNil(t, err)
@@ -531,6 +544,7 @@ func TestReset(t *testing.T) {
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			cs := NewMockSchemaColumnSetter(gomock.NewController(t))
 			ri := NewMockResourceInterface(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 			nsc2 := factory.Cache{}
 			s := &Store{
 				namespaceCache:   nsc,
@@ -538,12 +552,14 @@ func TestReset(t *testing.T) {
 				cacheFactory:     cf,
 				columnSetter:     cs,
 				cfInitializer:    func() (CacheFactory, error) { return cf, nil },
+				transformBuilder: tb,
 			}
 			nsSchema := baseNSSchema
 			cf.EXPECT().Reset().Return(nil)
 			cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
-			cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(nsc2, nil)
+			cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(nsc2, nil)
+			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			err := s.Reset()
 			assert.Nil(t, err)
 			assert.Equal(t, nsc2, s.namespaceCache)
@@ -556,6 +572,7 @@ func TestReset(t *testing.T) {
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			cs := NewMockSchemaColumnSetter(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
@@ -563,6 +580,7 @@ func TestReset(t *testing.T) {
 				cacheFactory:     cf,
 				columnSetter:     cs,
 				cfInitializer:    func() (CacheFactory, error) { return cf, nil },
+				transformBuilder: tb,
 			}
 
 			cf.EXPECT().Reset().Return(fmt.Errorf("error"))
@@ -577,6 +595,7 @@ func TestReset(t *testing.T) {
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			cs := NewMockSchemaColumnSetter(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
@@ -584,6 +603,7 @@ func TestReset(t *testing.T) {
 				cacheFactory:     cf,
 				columnSetter:     cs,
 				cfInitializer:    func() (CacheFactory, error) { return cf, nil },
+				transformBuilder: tb,
 			}
 
 			cf.EXPECT().Reset().Return(nil)
@@ -599,6 +619,7 @@ func TestReset(t *testing.T) {
 			cg := NewMockClientGetter(gomock.NewController(t))
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			cs := NewMockSchemaColumnSetter(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsi,
@@ -606,6 +627,7 @@ func TestReset(t *testing.T) {
 				cacheFactory:     cf,
 				columnSetter:     cs,
 				cfInitializer:    func() (CacheFactory, error) { return cf, nil },
+				transformBuilder: tb,
 			}
 			nsSchema := baseNSSchema
 
@@ -624,6 +646,7 @@ func TestReset(t *testing.T) {
 			cf := NewMockCacheFactory(gomock.NewController(t))
 			cs := NewMockSchemaColumnSetter(gomock.NewController(t))
 			ri := NewMockResourceInterface(gomock.NewController(t))
+			tb := NewMockTransformBuilder(gomock.NewController(t))
 
 			s := &Store{
 				namespaceCache:   nsc,
@@ -631,13 +654,15 @@ func TestReset(t *testing.T) {
 				cacheFactory:     cf,
 				columnSetter:     cs,
 				cfInitializer:    func() (CacheFactory, error) { return cf, nil },
+				transformBuilder: tb,
 			}
 			nsSchema := baseNSSchema
 
 			cf.EXPECT().Reset().Return(nil)
 			cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
-			cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
+			cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
+			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			err := s.Reset()
 			assert.NotNil(t, err)
 		},