#48673 - Added Timestamp Cache Handling to metadata.fields (#648)

* added timestamp convertion to metadata.fields

* fixed duration parsing

* fixed tests

* removed tags file

* added comments

* added better error handling

* changed ParseHumanDuration to use Fscanf

* added builtins handling

* adding mock updates

* fixing tests

* another try

* added timestamp convertion to metadata.fields

* addressing comments from @ericpromislow

* converting error to warning

* added template options

pkg/resources/common/duration.go

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+func ParseHumanReadableDuration(s string) (time.Duration, error) {
+	var total time.Duration
+	var val int
+	var unit byte
+
+	r := strings.NewReader(s)
+	for r.Len() > 0 {
+		if _, err := fmt.Fscanf(r, "%d%c", &val, &unit); err != nil {
+			return 0, fmt.Errorf("invalid duration in %s: %w", s, err)
+		}
+
+		switch unit {
+		case 'd':
+			total += time.Duration(val) * 24 * time.Hour
+		case 'h':
+			total += time.Duration(val) * time.Hour
+		case 'm':
+			total += time.Duration(val) * time.Minute
+		case 's':
+			total += time.Duration(val) * time.Second
+		default:
+			return 0, fmt.Errorf("invalid duration unit %s in %s", string(unit), s)
+		}
+	}
+
+	return total, nil
+}

pkg/resources/common/duration_test.go

@@ -0,0 +1,74 @@
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseHumanDuration(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    time.Duration
+		expectedErr bool
+	}{
+		{
+			name:     "days + hours + mins + secs",
+			input:    "1d23h45m56s",
+			expected: 24*time.Hour + 23*time.Hour + 45*time.Minute + 56*time.Second,
+		},
+		{
+			name:     "hours + mins + secs",
+			input:    "12h34m56s",
+			expected: 12*time.Hour + 34*time.Minute + 56*time.Second,
+		},
+		{
+			name:     "days + hours",
+			input:    "1d2h",
+			expected: 24*time.Hour + 2*time.Hour,
+		},
+		{
+			name:     "hours + secs",
+			input:    "1d2s",
+			expected: 24*time.Hour + 2*time.Second,
+		},
+		{
+			name:     "mins + secs",
+			input:    "1d2m",
+			expected: 24*time.Hour + 2*time.Minute,
+		},
+		{
+			name:     "hours",
+			input:    "1h",
+			expected: 1 * time.Hour,
+		},
+		{
+			name:     "mins",
+			input:    "1m",
+			expected: 1 * time.Minute,
+		},
+		{
+			name:     "secs",
+			input:    "0s",
+			expected: 0 * time.Second,
+		},
+		{
+			name:        "invalid input",
+			input:       "<invalid>",
+			expectedErr: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			output, err := ParseHumanReadableDuration(tc.input)
+			if tc.expectedErr {
+				assert.Error(t, err)
+			} else {
+				assert.Equal(t, tc.expected, output)
+			}
+		})
+	}
+}

pkg/resources/common/formatter.go

@@ -2,12 +2,17 @@ package common
 
 import (
 	"net/http"
+	"slices"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/rancher/apiserver/pkg/types"
 	"github.com/rancher/steve/pkg/accesscontrol"
 	"github.com/rancher/steve/pkg/attributes"
 	"github.com/rancher/steve/pkg/resources/virtual/common"
+	"github.com/sirupsen/logrus"
+
 	"github.com/rancher/steve/pkg/schema"
 	metricsStore "github.com/rancher/steve/pkg/stores/metrics"
 	"github.com/rancher/steve/pkg/stores/proxy"
@@ -19,25 +24,32 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	schema2 "k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/duration"
 )
 
+type TemplateOptions struct {
+	InSQLMode bool
+}
+
 func DefaultTemplate(clientGetter proxy.ClientGetter,
 	summaryCache *summarycache.SummaryCache,
 	asl accesscontrol.AccessSetLookup,
-	namespaceCache corecontrollers.NamespaceCache) schema.Template {
+	namespaceCache corecontrollers.NamespaceCache,
+	options TemplateOptions) schema.Template {
 	return schema.Template{
 		Store:     metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)),
-		Formatter: formatter(summaryCache, asl),
+		Formatter: formatter(summaryCache, asl, options),
 	}
 }
 
 // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
 func DefaultTemplateForStore(store types.Store,
 	summaryCache *summarycache.SummaryCache,
-	asl accesscontrol.AccessSetLookup) schema.Template {
+	asl accesscontrol.AccessSetLookup,
+	options TemplateOptions) schema.Template {
 	return schema.Template{
 		Store:     store,
-		Formatter: formatter(summaryCache, asl),
+		Formatter: formatter(summaryCache, asl, options),
 	}
 }
 
@@ -83,7 +95,7 @@ func buildBasePath(gvr schema2.GroupVersionResource, namespace string, includeNa
 	return buf.String()
 }
 
-func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup) types.Formatter {
+func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup, options TemplateOptions) types.Formatter {
 	return func(request *types.APIRequest, resource *types.RawResource) {
 		if resource.Schema == nil {
 			return
@@ -140,6 +152,7 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook
 			delete(resource.Links, "patch")
 		}
 
+		gvk := attributes.GVK(resource.Schema)
 		if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok {
 			// with the sql cache, these were already added by the indexer. However, the sql cache
 			// is only used for lists, so we need to re-add here for get/watch
@@ -157,6 +170,10 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook
 			includeFields(request, unstr)
 			excludeFields(request, unstr)
 			excludeValues(request, unstr)
+
+			if options.InSQLMode {
+				convertMetadataTimestampFields(request, gvk, unstr)
+			}
 		}
 
 		if permsQuery := request.Query.Get("checkPermissions"); permsQuery != "" {
@@ -212,6 +229,56 @@ func excludeFields(request *types.APIRequest, unstr *unstructured.Unstructured)
 	}
 }
 
+// convertMetadataTimestampFields updates metadata timestamp fields to ensure they remain fresh and human-readable when sent back
+// to the client. Internally, fields are stored as Unix timestamps; on each request, we calculate the elapsed time since
+// those timestamps by subtracting them from time.Now(), then format the resulting duration into a human-friendly string.
+// This prevents cached durations (e.g. “2d” - 2 days) from becoming stale over time.
+func convertMetadataTimestampFields(request *types.APIRequest, gvk schema2.GroupVersionKind, unstr *unstructured.Unstructured) {
+	if request.Schema != nil {
+		cols := GetColumnDefinitions(request.Schema)
+		for _, col := range cols {
+			gvkDateFields, gvkFound := DateFieldsByGVKBuiltins[gvk]
+			if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) {
+				index := GetIndexValueFromString(col.Field)
+				if index == -1 {
+					logrus.Errorf("field index not found at column.Field struct variable: %s", col.Field)
+					return
+				}
+
+				curValue, got, err := unstructured.NestedSlice(unstr.Object, "metadata", "fields")
+				if err != nil {
+					logrus.Errorf("failed to get metadata.fields slice from unstr.Object: %s", err.Error())
+				}
+
+				if !got {
+					logrus.Debugf("couldn't find metadata.fields at unstr.Object")
+					return
+				}
+
+				timeValue, ok := curValue[index].(string)
+				if !ok {
+					logrus.Debugf("time field isn't a string")
+					return
+				}
+				millis, err := strconv.ParseInt(timeValue, 10, 64)
+				if err != nil {
+					logrus.Warnf("failed to convert timestamp value: %s", err.Error())
+					return
+				}
+
+				timestamp := time.Unix(0, millis*int64(time.Millisecond))
+				dur := time.Since(timestamp)
+
+				curValue[index] = duration.HumanDuration(dur)
+				if err := unstructured.SetNestedSlice(unstr.Object, curValue, "metadata", "fields"); err != nil {
+					logrus.Errorf("failed to set value back to metadata.fields slice: %s", err.Error())
+					return
+				}
+			}
+		}
+	}
+}
+
 func excludeValues(request *types.APIRequest, unstr *unstructured.Unstructured) {
 	if values, ok := request.Query["excludeValues"]; ok {
 		for _, f := range values {

pkg/resources/common/formatter_test.go

@@ -1068,7 +1068,7 @@ func Test_formatterLinks(t *testing.T) {
 				APIObject: test.apiObject,
 				Links:     test.currentLinks,
 			}
-			fmtter := formatter(nil, asl)
+			fmtter := formatter(nil, asl, TemplateOptions{InSQLMode: false})
 			fmtter(request, resource)
 			require.Equal(t, test.wantLinks, resource.Links)
 
@@ -1269,7 +1269,7 @@ func TestFormatterAddsResourcePermissions(t *testing.T) {
 
 			asl.EXPECT().AccessFor(&defaultUserInfo).Return(&accessSet).AnyTimes()
 
-			formatter := formatter(fakeCache, asl)
+			formatter := formatter(fakeCache, asl, TemplateOptions{InSQLMode: false})
 			formatter(req, resource)
 
 			// Extract the resultant resourcePermissions

pkg/resources/common/gvkdatefields.go

@@ -0,0 +1,78 @@
+package common
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var DateFieldsByGVKBuiltins = map[schema.GroupVersionKind][]string{
+	{Group: "", Version: "v1", Kind: "ConfigMap"}:             {"Age"},
+	{Group: "", Version: "v1", Kind: "Endpoints"}:             {"Age"},
+	{Group: "", Version: "v1", Kind: "Event"}:                 {"Last Seen", "First Seen"},
+	{Group: "", Version: "v1", Kind: "Namespace"}:             {"Age"},
+	{Group: "", Version: "v1", Kind: "Node"}:                  {"Age"},
+	{Group: "", Version: "v1", Kind: "PersistentVolume"}:      {"Age"},
+	{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}: {"Age"},
+	{Group: "", Version: "v1", Kind: "Pod"}:                   {"Age"},
+	{Group: "", Version: "v1", Kind: "ReplicationController"}: {"Age"},
+	{Group: "", Version: "v1", Kind: "ResourceQuota"}:         {"Age"},
+	{Group: "", Version: "v1", Kind: "Secret"}:                {"Age"},
+	{Group: "", Version: "v1", Kind: "Service"}:               {"Age"},
+	{Group: "", Version: "v1", Kind: "ServiceAccount"}:        {"Age"},
+
+	{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicy"}:          {"Age"},
+	{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicyBinding"}:   {"Age"},
+	{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicy"}:        {"Age"},
+	{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicyBinding"}: {"Age"},
+	{Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "MutatingWebhookConfiguration"}:      {"Age"},
+	{Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "ValidatingWebhookConfiguration"}:    {"Age"},
+
+	{Group: "apiserverinternal.k8s.io", Version: "v1alpha1", Kind: "StorageVersion"}: {"Age"},
+
+	{Group: "apps", Version: "v1", Kind: "DaemonSet"}:               {"Age"},
+	{Group: "apps", Version: "v1", Kind: "Deployment"}:              {"Age"},
+	{Group: "apps", Version: "v1", Kind: "ReplicaSet"}:              {"Age"},
+	{Group: "apps", Version: "v1", Kind: "StatefulSet"}:             {"Age"},
+	{Group: "apps", Version: "v1beta1", Kind: "ControllerRevision"}: {"Age"},
+
+	{Group: "autoscaling", Version: "v1", Kind: "Scale"}:                        {"Age"},
+	{Group: "autoscaling", Version: "v2beta1", Kind: "HorizontalPodAutoscaler"}: {"Age"},
+
+	{Group: "batch", Version: "v1", Kind: "Job"}:          {"Duration", "Age"},
+	{Group: "batch", Version: "v1beta1", Kind: "CronJob"}: {"Last Schedule", "Age"},
+
+	{Group: "certificates.k8s.io", Version: "v1beta1", Kind: "CertificateSigningRequest"}: {"Age"},
+
+	{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}:                {"Age"},
+	{Group: "coordination.k8s.io", Version: "v1alpha2", Kind: "LeaseCandidate"}: {"Age"},
+
+	{Group: "discovery.k8s.io", Version: "v1beta1", Kind: "EndpointSlice"}: {"Age"},
+
+	{Group: "flowcontrol.k8s.io", Version: "v1", Kind: "FlowSchema"}:                 {"Age"},
+	{Group: "flowcontrol.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}: {"Age"},
+
+	{Group: "networking.k8s.io", Version: "v1beta1", Kind: "Ingress"}:       {"Age"},
+	{Group: "networking.k8s.io", Version: "v1beta1", Kind: "IngressClass"}:  {"Age"},
+	{Group: "networking.k8s.io", Version: "v1beta1", Kind: "IPAddress"}:     {"Age"},
+	{Group: "networking.k8s.io", Version: "v1beta1", Kind: "NetworkPolicy"}: {"Age"},
+	{Group: "networking.k8s.io", Version: "v1beta1", Kind: "ServiceCIDR"}:   {"Age"},
+
+	{Group: "node.k8s.io", Version: "v1", Kind: "RuntimeClass"}: {"Age"},
+
+	{Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"}: {"Age"},
+
+	{Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "ClusterRoleBinding"}: {"Age"},
+	{Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "RoleBinding"}:        {"Age"},
+
+	{Group: "resource.k8s.io", Version: "v1beta1", Kind: "DeviceClass"}:           {"Age"},
+	{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaim"}:         {"Age"},
+	{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaimTemplate"}: {"Age"},
+	{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceSlice"}:         {"Age"},
+
+	{Group: "scheduling.k8s.io", Version: "v1", Kind: "PriorityClass"}: {"Age"},
+
+	{Group: "storage.k8s.io", Version: "v1", Kind: "CSIDriver"}:                  {"Age"},
+	{Group: "storage.k8s.io", Version: "v1", Kind: "CSINode"}:                    {"Age"},
+	{Group: "storage.k8s.io", Version: "v1", Kind: "StorageClass"}:               {"Age"},
+	{Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"}:           {"Age"},
+	{Group: "storage.k8s.io", Version: "v1beta1", Kind: "VolumeAttributesClass"}: {"Age"},
+}

pkg/resources/common/util.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/rancher/apiserver/pkg/types"
+	"github.com/rancher/steve/pkg/attributes"
+)
+
+// GetIndexValueFromString looks for values between [ ].
+// e.g: $.metadata.fields[2], in this case it would return 2
+// In case it doesn't find any value between brackets it returns -1
+func GetIndexValueFromString(pathString string) int {
+	idxStart := strings.Index(pathString, "[")
+	if idxStart == -1 {
+		return -1
+	}
+	idxEnd := strings.Index(pathString[idxStart+1:], "]")
+	if idxEnd == -1 {
+		return -1
+	}
+	idx, err := strconv.Atoi(pathString[idxStart+1 : idxStart+1+idxEnd])
+	if err != nil {
+		return -1
+	}
+	return idx
+}
+
+// GetColumnDefinitions returns ColumnDefinitions from an APISchema
+func GetColumnDefinitions(schema *types.APISchema) []ColumnDefinition {
+	columns := attributes.Columns(schema)
+	if columns == nil {
+		return nil
+	}
+	colDefs, ok := columns.([]ColumnDefinition)
+	if !ok {
+		return nil
+	}
+	return colDefs
+}

pkg/resources/schema.go

@@ -47,9 +47,10 @@ func DefaultSchemaTemplates(cf *client.Factory,
 	summaryCache *summarycache.SummaryCache,
 	lookup accesscontrol.AccessSetLookup,
 	discovery discovery.DiscoveryInterface,
-	namespaceCache corecontrollers.NamespaceCache) []schema.Template {
+	namespaceCache corecontrollers.NamespaceCache,
+	options common.TemplateOptions) []schema.Template {
 	return []schema.Template{
-		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache),
+		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, options),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",
@@ -77,10 +78,11 @@ func DefaultSchemaTemplatesForStore(store types.Store,
 	baseSchemas *types.APISchemas,
 	summaryCache *summarycache.SummaryCache,
 	lookup accesscontrol.AccessSetLookup,
-	discovery discovery.DiscoveryInterface) []schema.Template {
+	discovery discovery.DiscoveryInterface,
+	options common.TemplateOptions) []schema.Template {
 
 	return []schema.Template{
-		common.DefaultTemplateForStore(store, summaryCache, lookup),
+		common.DefaultTemplateForStore(store, summaryCache, lookup, options),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",

pkg/resources/virtual/virtual.go

@@ -4,7 +4,10 @@ package virtual
 
 import (
 	"fmt"
+	"slices"
+	"time"
 
+	rescommon "github.com/rancher/steve/pkg/resources/common"
 	"github.com/rancher/steve/pkg/resources/virtual/clusters"
 	"github.com/rancher/steve/pkg/resources/virtual/common"
 	"github.com/rancher/steve/pkg/resources/virtual/events"
@@ -13,6 +16,8 @@ import (
 	"k8s.io/client-go/tools/cache"
 )
 
+var now = time.Now()
+
 // TransformBuilder builds transform functions for specified GVKs through GetTransformFunc
 type TransformBuilder struct {
 	defaultFields *common.DefaultFields
@@ -28,13 +33,52 @@ func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder {
 }
 
 // GetTransformFunc returns the func to transform a raw object into a fixed object, if needed
-func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc {
+func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind, columns []rescommon.ColumnDefinition) cache.TransformFunc {
 	converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0)
 	if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" {
 		converters = append(converters, events.TransformEventObject)
 	} else if gvk.Kind == "Cluster" && gvk.Group == "management.cattle.io" && gvk.Version == "v3" {
 		converters = append(converters, clusters.TransformManagedCluster)
 	}
+
+	// Detecting if we need to convert date fields
+	for _, col := range columns {
+		gvkDateFields, gvkFound := rescommon.DateFieldsByGVKBuiltins[gvk]
+		if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) {
+			converters = append(converters, func(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
+				index := rescommon.GetIndexValueFromString(col.Field)
+				if index == -1 {
+					return nil, fmt.Errorf("field index not found at column.Field struct variable: %s", col.Field)
+				}
+
+				curValue, got, err := unstructured.NestedSlice(obj.Object, "metadata", "fields")
+				if !got {
+					return obj, nil
+				}
+
+				if err != nil {
+					return nil, err
+				}
+
+				value, cast := curValue[index].(string)
+				if !cast {
+					return nil, fmt.Errorf("could not cast metadata.fields %d to string", index)
+				}
+				duration, err := rescommon.ParseHumanReadableDuration(value)
+				if err != nil {
+					return nil, err
+				}
+
+				curValue[index] = fmt.Sprintf("%d", now.Add(-duration).UnixMilli())
+				if err := unstructured.SetNestedSlice(obj.Object, curValue, "metadata", "fields"); err != nil {
+					return nil, err
+				}
+
+				return obj, nil
+			})
+		}
+	}
+
 	converters = append(converters, t.defaultFields.TransformCommon)
 
 	return func(raw interface{}) (interface{}, error) {

pkg/resources/virtual/virtual_test.go

@@ -1,11 +1,12 @@
-package virtual_test
+package virtual
 
 import (
 	"fmt"
 	"strings"
 	"testing"
+	"time"
 
-	"github.com/rancher/steve/pkg/resources/virtual"
+	rescommon "github.com/rancher/steve/pkg/resources/common"
 	"github.com/rancher/steve/pkg/resources/virtual/common"
 	"github.com/rancher/steve/pkg/summarycache"
 	"github.com/rancher/wrangler/v3/pkg/summary"
@@ -16,11 +17,14 @@ import (
 )
 
 func TestTransformChain(t *testing.T) {
+	now = func() time.Time { return time.Date(1992, 9, 2, 0, 0, 0, 0, time.UTC) }()
+	noColumns := []rescommon.ColumnDefinition{}
 	tests := []struct {
 		name             string
 		input            any
 		hasSummary       *summary.SummarizedObject
 		hasRelationships []summarycache.Relationship
+		columns          []rescommon.ColumnDefinition
 		wantOutput       any
 		wantError        bool
 	}{
@@ -65,6 +69,7 @@ func TestTransformChain(t *testing.T) {
 					"id": "old-id",
 				},
 			},
+			columns: noColumns,
 			wantOutput: &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "test.cattle.io/v1",
@@ -94,6 +99,131 @@ func TestTransformChain(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "CRD metadata.fields has a date field - should convert to timestamp",
+			hasSummary: &summary.SummarizedObject{
+				PartialObjectMetadata: v1.PartialObjectMetadata{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "testobj",
+						Namespace: "test-ns",
+					},
+					TypeMeta: v1.TypeMeta{
+						APIVersion: "test.cattle.io/v1",
+						Kind:       "TestResource",
+					},
+				},
+				Summary: summary.Summary{
+					State:         "success",
+					Transitioning: false,
+					Error:         false,
+				},
+			},
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+						"fields":    []interface{}{"1d"},
+					},
+					"id": "old-id",
+				},
+			},
+			columns: []rescommon.ColumnDefinition{
+				{
+					Field: "metadata.fields[0]",
+					TableColumnDefinition: v1.TableColumnDefinition{
+						Name: "Age",
+						Type: "date",
+					},
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "test.cattle.io/v1",
+					"kind":       "TestResource",
+					"metadata": map[string]interface{}{
+						"name":          "testobj",
+						"namespace":     "test-ns",
+						"relationships": []any(nil),
+						"state": map[string]interface{}{
+							"name":          "success",
+							"error":         false,
+							"transitioning": false,
+							"message":       "",
+						},
+						"fields": []interface{}{
+							fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()),
+						},
+					},
+					"id":  "test-ns/testobj",
+					"_id": "old-id",
+				},
+			},
+		},
+		{
+			name: "built-in type metadata.fields has a date field - should convert to timestamp",
+			hasSummary: &summary.SummarizedObject{
+				PartialObjectMetadata: v1.PartialObjectMetadata{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "testobj",
+						Namespace: "test-ns",
+					},
+					TypeMeta: v1.TypeMeta{
+						APIVersion: "apps/v1",
+						Kind:       "Deployment",
+					},
+				},
+				Summary: summary.Summary{
+					State:         "success",
+					Transitioning: false,
+					Error:         false,
+				},
+			},
+			input: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "testobj",
+						"namespace": "test-ns",
+						"fields":    []interface{}{"1d"},
+					},
+					"id": "old-id",
+				},
+			},
+			columns: []rescommon.ColumnDefinition{
+				{
+					TableColumnDefinition: v1.TableColumnDefinition{
+						Name: "Age",
+					},
+					Field: "metadata.fields[0]",
+				},
+			},
+			wantOutput: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":          "testobj",
+						"namespace":     "test-ns",
+						"relationships": []any(nil),
+						"state": map[string]interface{}{
+							"name":          "success",
+							"error":         false,
+							"transitioning": false,
+							"message":       "",
+						},
+						"fields": []interface{}{
+							fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()),
+						},
+					},
+					"id":  "test-ns/testobj",
+					"_id": "old-id",
+				},
+			},
+		},
 		{
 			name: "processable event",
 			input: &unstructured.Unstructured{
@@ -118,6 +248,7 @@ func TestTransformChain(t *testing.T) {
 					"type": "Gorniplatz",
 				},
 			},
+			columns: noColumns,
 			wantOutput: &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "/v1",
@@ -162,6 +293,7 @@ func TestTransformChain(t *testing.T) {
 					"type": "Gorniplatz",
 				},
 			},
+			columns: noColumns,
 			wantOutput: &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "palau.io/v1",
@@ -218,6 +350,7 @@ func TestTransformChain(t *testing.T) {
 					},
 				},
 			},
+			columns: noColumns,
 			wantOutput: &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "management.cattle.io/v3",
@@ -302,6 +435,7 @@ func TestTransformChain(t *testing.T) {
 					},
 				},
 			},
+			columns: noColumns,
 			wantOutput: &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "management.cattle.io/v3",
@@ -346,13 +480,14 @@ func TestTransformChain(t *testing.T) {
 			},
 		},
 	}
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			fakeCache := common.FakeSummaryCache{
 				SummarizedObject: test.hasSummary,
 				Relationships:    test.hasRelationships,
 			}
-			tb := virtual.NewTransformBuilder(&fakeCache)
+			tb := NewTransformBuilder(&fakeCache)
 			raw, isSignal, err := common.GetUnstructured(test.input)
 			require.False(t, isSignal)
 			require.Nil(t, err)
@@ -362,7 +497,7 @@ func TestTransformChain(t *testing.T) {
 			if test.name == "a non-ready cluster" {
 				fmt.Printf("Stop here")
 			}
-			output, err := tb.GetTransformFunc(gvk)(test.input)
+			output, err := tb.GetTransformFunc(gvk, test.columns)(test.input)
 			require.Equal(t, test.wantOutput, output)
 			if test.wantError {
 				require.Error(t, err)

pkg/server/server.go

@@ -224,7 +224,7 @@ func setup(ctx context.Context, server *Server) error {
 		store := metricsStore.NewMetricsStore(errStore)
 		// end store setup code
 
-		for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery()) {
+		for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), common.TemplateOptions{InSQLMode: true}) {
 			sf.AddTemplate(template)
 		}
 
@@ -238,7 +238,7 @@ func setup(ctx context.Context, server *Server) error {
 			return nil
 		}
 	} else {
-		for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache()) {
+		for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache(), common.TemplateOptions{InSQLMode: false}) {
 			sf.AddTemplate(template)
 		}
 		onSchemasHandler = ccache.OnSchemas

pkg/stores/sqlproxy/proxy_mocks_test.go

@@ -14,6 +14,7 @@ import (
 	reflect "reflect"
 
 	types "github.com/rancher/apiserver/pkg/types"
+	common "github.com/rancher/steve/pkg/resources/common"
 	factory "github.com/rancher/steve/pkg/sqlcache/informer/factory"
 	partition "github.com/rancher/steve/pkg/sqlcache/partition"
 	sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes"
@@ -389,15 +390,15 @@ func (m *MockTransformBuilder) EXPECT() *MockTransformBuilderMockRecorder {
 }
 
 // GetTransformFunc mocks base method.
-func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind) cache.TransformFunc {
+func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind, arg1 []common.ColumnDefinition) cache.TransformFunc {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetTransformFunc", arg0)
+	ret := m.ctrl.Call(m, "GetTransformFunc", arg0, arg1)
 	ret0, _ := ret[0].(cache.TransformFunc)
 	return ret0
 }
 
 // GetTransformFunc indicates an expected call of GetTransformFunc.
-func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0 any) *gomock.Call {
+func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0, arg1)
 }

pkg/stores/sqlproxy/proxy_store.go

@@ -240,7 +240,7 @@ type RelationshipNotifier interface {
 }
 
 type TransformBuilder interface {
-	GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc
+	GetTransformFunc(gvk schema.GroupVersionKind, colDefs []common.ColumnDefinition) cache.TransformFunc
 }
 
 type Store struct {
@@ -335,9 +335,10 @@ func (s *Store) initializeNamespaceCache() error {
 
 	// get any type-specific fields that steve is interested in
 	fields = append(fields, getFieldForGVK(gvk)...)
+	cols := common.GetColumnDefinitions(&nsSchema)
 
 	// get the type-specific transform func
-	transformFunc := s.transformBuilder.GetTransformFunc(gvk)
+	transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols)
 
 	// get the ns informer
 	tableClient := &tablelistconvert.Client{ResourceInterface: client}
@@ -545,7 +546,7 @@ func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types.
 	gvk := attributes.GVK(schema)
 	fields := getFieldsFromSchema(schema)
 	fields = append(fields, getFieldForGVK(gvk)...)
-	transformFunc := s.transformBuilder.GetTransformFunc(gvk)
+	transformFunc := s.transformBuilder.GetTransformFunc(gvk, nil)
 	tableClient := &tablelistconvert.Client{ResourceInterface: client}
 	attrs := attributes.GVK(schema)
 	ns := attributes.Namespaced(schema)
@@ -756,7 +757,9 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, apiSchema *types.APISc
 	gvk := attributes.GVK(apiSchema)
 	fields := getFieldsFromSchema(apiSchema)
 	fields = append(fields, getFieldForGVK(gvk)...)
-	transformFunc := s.transformBuilder.GetTransformFunc(gvk)
+	cols := common.GetColumnDefinitions(apiSchema)
+
+	transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols)
 	tableClient := &tablelistconvert.Client{ResourceInterface: client}
 	attrs := attributes.GVK(apiSchema)
 	ns := attributes.Namespaced(apiSchema)

pkg/stores/sqlproxy/proxy_store_test.go

@@ -245,7 +245,7 @@ func TestListByPartitions(t *testing.T) {
 			cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
 			// This tests that fields are being extracted from schema columns and the type specific fields map
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
-			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
 			list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
 			assert.Nil(t, err)
@@ -462,7 +462,7 @@ func TestListByPartitions(t *testing.T) {
 			// note also the watchable bool is expected to be false
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil)
 
-			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
 			list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
 			assert.Nil(t, err)
@@ -534,7 +534,7 @@ func TestListByPartitions(t *testing.T) {
 			assert.Nil(t, err)
 			cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
 			// This tests that fields are being extracted from schema columns and the type specific fields map
-			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error"))
 
 			_, _, _, err = s.ListByPartitions(req, schema, partitions)
@@ -613,7 +613,7 @@ func TestListByPartitions(t *testing.T) {
 			// This tests that fields are being extracted from schema columns and the type specific fields map
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
 			bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error"))
-			tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 
 			_, _, _, err = s.ListByPartitions(req, schema, partitions)
 			assert.NotNil(t, err)
@@ -725,7 +725,7 @@ func TestListByPartitionWithUserAccess(t *testing.T) {
 			attributes.SetGVK(theSchema, gvk)
 			cg.EXPECT().TableAdminClient(apiOp, theSchema, "", &WarningBuffer{}).Return(ri, nil)
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {"id"}, {"metadata", "state", "name"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(theSchema), attributes.Namespaced(theSchema), true).Return(c, nil)
-			tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 
 			listToReturn := &unstructured.UnstructuredList{
 				Items: make([]unstructured.Unstructured, 0, 0),
@@ -767,7 +767,7 @@ func TestReset(t *testing.T) {
 			cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil)
-			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			err := s.Reset()
 			assert.Nil(t, err)
 			assert.Equal(t, nsc2, s.namespaceCache)
@@ -874,7 +874,7 @@ func TestReset(t *testing.T) {
 			cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
 			cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
 			cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error"))
-			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
+			tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
 			err := s.Reset()
 			assert.NotNil(t, err)
 		},