diff --git a/pkg/resources/common/duration.go b/pkg/resources/common/duration.go new file mode 100644 index 00000000..b4f5c9f0 --- /dev/null +++ b/pkg/resources/common/duration.go @@ -0,0 +1,35 @@ +package common + +import ( + "fmt" + "strings" + "time" +) + +func ParseHumanReadableDuration(s string) (time.Duration, error) { + var total time.Duration + var val int + var unit byte + + r := strings.NewReader(s) + for r.Len() > 0 { + if _, err := fmt.Fscanf(r, "%d%c", &val, &unit); err != nil { + return 0, fmt.Errorf("invalid duration in %s: %w", s, err) + } + + switch unit { + case 'd': + total += time.Duration(val) * 24 * time.Hour + case 'h': + total += time.Duration(val) * time.Hour + case 'm': + total += time.Duration(val) * time.Minute + case 's': + total += time.Duration(val) * time.Second + default: + return 0, fmt.Errorf("invalid duration unit %s in %s", string(unit), s) + } + } + + return total, nil +} diff --git a/pkg/resources/common/duration_test.go b/pkg/resources/common/duration_test.go new file mode 100644 index 00000000..6091cc31 --- /dev/null +++ b/pkg/resources/common/duration_test.go @@ -0,0 +1,74 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseHumanDuration(t *testing.T) { + testCases := []struct { + name string + input string + expected time.Duration + expectedErr bool + }{ + { + name: "days + hours + mins + secs", + input: "1d23h45m56s", + expected: 24*time.Hour + 23*time.Hour + 45*time.Minute + 56*time.Second, + }, + { + name: "hours + mins + secs", + input: "12h34m56s", + expected: 12*time.Hour + 34*time.Minute + 56*time.Second, + }, + { + name: "days + hours", + input: "1d2h", + expected: 24*time.Hour + 2*time.Hour, + }, + { + name: "hours + secs", + input: "1d2s", + expected: 24*time.Hour + 2*time.Second, + }, + { + name: "mins + secs", + input: "1d2m", + expected: 24*time.Hour + 2*time.Minute, + }, + { + name: "hours", + input: "1h", + expected: 1 * time.Hour, + }, + { + name: "mins", + input: "1m", + expected: 1 * time.Minute, + }, + { + name: "secs", + input: "0s", + expected: 0 * time.Second, + }, + { + name: "invalid input", + input: "", + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output, err := ParseHumanReadableDuration(tc.input) + if tc.expectedErr { + assert.Error(t, err) + } else { + assert.Equal(t, tc.expected, output) + } + }) + } +} diff --git a/pkg/resources/common/formatter.go b/pkg/resources/common/formatter.go index 5171b623..8f9bd72d 100644 --- a/pkg/resources/common/formatter.go +++ b/pkg/resources/common/formatter.go @@ -2,12 +2,17 @@ package common import ( "net/http" + "slices" + "strconv" "strings" + "time" "github.com/rancher/apiserver/pkg/types" "github.com/rancher/steve/pkg/accesscontrol" "github.com/rancher/steve/pkg/attributes" "github.com/rancher/steve/pkg/resources/virtual/common" + "github.com/sirupsen/logrus" + "github.com/rancher/steve/pkg/schema" metricsStore "github.com/rancher/steve/pkg/stores/metrics" "github.com/rancher/steve/pkg/stores/proxy" @@ -19,25 +24,32 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" schema2 "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" ) +type TemplateOptions struct { + InSQLMode bool +} + func DefaultTemplate(clientGetter proxy.ClientGetter, summaryCache *summarycache.SummaryCache, asl accesscontrol.AccessSetLookup, - namespaceCache corecontrollers.NamespaceCache) schema.Template { + namespaceCache corecontrollers.NamespaceCache, + options TemplateOptions) schema.Template { return schema.Template{ Store: metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)), - Formatter: formatter(summaryCache, asl), + Formatter: formatter(summaryCache, asl, options), } } // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally. func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache, - asl accesscontrol.AccessSetLookup) schema.Template { + asl accesscontrol.AccessSetLookup, + options TemplateOptions) schema.Template { return schema.Template{ Store: store, - Formatter: formatter(summaryCache, asl), + Formatter: formatter(summaryCache, asl, options), } } @@ -83,7 +95,7 @@ func buildBasePath(gvr schema2.GroupVersionResource, namespace string, includeNa return buf.String() } -func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup) types.Formatter { +func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup, options TemplateOptions) types.Formatter { return func(request *types.APIRequest, resource *types.RawResource) { if resource.Schema == nil { return @@ -140,6 +152,7 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook delete(resource.Links, "patch") } + gvk := attributes.GVK(resource.Schema) if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok { // with the sql cache, these were already added by the indexer. However, the sql cache // is only used for lists, so we need to re-add here for get/watch @@ -157,6 +170,10 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook includeFields(request, unstr) excludeFields(request, unstr) excludeValues(request, unstr) + + if options.InSQLMode { + convertMetadataTimestampFields(request, gvk, unstr) + } } if permsQuery := request.Query.Get("checkPermissions"); permsQuery != "" { @@ -212,6 +229,56 @@ func excludeFields(request *types.APIRequest, unstr *unstructured.Unstructured) } } +// convertMetadataTimestampFields updates metadata timestamp fields to ensure they remain fresh and human-readable when sent back +// to the client. Internally, fields are stored as Unix timestamps; on each request, we calculate the elapsed time since +// those timestamps by subtracting them from time.Now(), then format the resulting duration into a human-friendly string. +// This prevents cached durations (e.g. “2d” - 2 days) from becoming stale over time. +func convertMetadataTimestampFields(request *types.APIRequest, gvk schema2.GroupVersionKind, unstr *unstructured.Unstructured) { + if request.Schema != nil { + cols := GetColumnDefinitions(request.Schema) + for _, col := range cols { + gvkDateFields, gvkFound := DateFieldsByGVKBuiltins[gvk] + if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) { + index := GetIndexValueFromString(col.Field) + if index == -1 { + logrus.Errorf("field index not found at column.Field struct variable: %s", col.Field) + return + } + + curValue, got, err := unstructured.NestedSlice(unstr.Object, "metadata", "fields") + if err != nil { + logrus.Errorf("failed to get metadata.fields slice from unstr.Object: %s", err.Error()) + } + + if !got { + logrus.Debugf("couldn't find metadata.fields at unstr.Object") + return + } + + timeValue, ok := curValue[index].(string) + if !ok { + logrus.Debugf("time field isn't a string") + return + } + millis, err := strconv.ParseInt(timeValue, 10, 64) + if err != nil { + logrus.Warnf("failed to convert timestamp value: %s", err.Error()) + return + } + + timestamp := time.Unix(0, millis*int64(time.Millisecond)) + dur := time.Since(timestamp) + + curValue[index] = duration.HumanDuration(dur) + if err := unstructured.SetNestedSlice(unstr.Object, curValue, "metadata", "fields"); err != nil { + logrus.Errorf("failed to set value back to metadata.fields slice: %s", err.Error()) + return + } + } + } + } +} + func excludeValues(request *types.APIRequest, unstr *unstructured.Unstructured) { if values, ok := request.Query["excludeValues"]; ok { for _, f := range values { diff --git a/pkg/resources/common/formatter_test.go b/pkg/resources/common/formatter_test.go index d0122efa..9eb8b06e 100644 --- a/pkg/resources/common/formatter_test.go +++ b/pkg/resources/common/formatter_test.go @@ -1068,7 +1068,7 @@ func Test_formatterLinks(t *testing.T) { APIObject: test.apiObject, Links: test.currentLinks, } - fmtter := formatter(nil, asl) + fmtter := formatter(nil, asl, TemplateOptions{InSQLMode: false}) fmtter(request, resource) require.Equal(t, test.wantLinks, resource.Links) @@ -1269,7 +1269,7 @@ func TestFormatterAddsResourcePermissions(t *testing.T) { asl.EXPECT().AccessFor(&defaultUserInfo).Return(&accessSet).AnyTimes() - formatter := formatter(fakeCache, asl) + formatter := formatter(fakeCache, asl, TemplateOptions{InSQLMode: false}) formatter(req, resource) // Extract the resultant resourcePermissions diff --git a/pkg/resources/common/gvkdatefields.go b/pkg/resources/common/gvkdatefields.go new file mode 100644 index 00000000..4d024be7 --- /dev/null +++ b/pkg/resources/common/gvkdatefields.go @@ -0,0 +1,78 @@ +package common + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var DateFieldsByGVKBuiltins = map[schema.GroupVersionKind][]string{ + {Group: "", Version: "v1", Kind: "ConfigMap"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Endpoints"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Event"}: {"Last Seen", "First Seen"}, + {Group: "", Version: "v1", Kind: "Namespace"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Node"}: {"Age"}, + {Group: "", Version: "v1", Kind: "PersistentVolume"}: {"Age"}, + {Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Pod"}: {"Age"}, + {Group: "", Version: "v1", Kind: "ReplicationController"}: {"Age"}, + {Group: "", Version: "v1", Kind: "ResourceQuota"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Secret"}: {"Age"}, + {Group: "", Version: "v1", Kind: "Service"}: {"Age"}, + {Group: "", Version: "v1", Kind: "ServiceAccount"}: {"Age"}, + + {Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicy"}: {"Age"}, + {Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicyBinding"}: {"Age"}, + {Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicy"}: {"Age"}, + {Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicyBinding"}: {"Age"}, + {Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "MutatingWebhookConfiguration"}: {"Age"}, + {Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "ValidatingWebhookConfiguration"}: {"Age"}, + + {Group: "apiserverinternal.k8s.io", Version: "v1alpha1", Kind: "StorageVersion"}: {"Age"}, + + {Group: "apps", Version: "v1", Kind: "DaemonSet"}: {"Age"}, + {Group: "apps", Version: "v1", Kind: "Deployment"}: {"Age"}, + {Group: "apps", Version: "v1", Kind: "ReplicaSet"}: {"Age"}, + {Group: "apps", Version: "v1", Kind: "StatefulSet"}: {"Age"}, + {Group: "apps", Version: "v1beta1", Kind: "ControllerRevision"}: {"Age"}, + + {Group: "autoscaling", Version: "v1", Kind: "Scale"}: {"Age"}, + {Group: "autoscaling", Version: "v2beta1", Kind: "HorizontalPodAutoscaler"}: {"Age"}, + + {Group: "batch", Version: "v1", Kind: "Job"}: {"Duration", "Age"}, + {Group: "batch", Version: "v1beta1", Kind: "CronJob"}: {"Last Schedule", "Age"}, + + {Group: "certificates.k8s.io", Version: "v1beta1", Kind: "CertificateSigningRequest"}: {"Age"}, + + {Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}: {"Age"}, + {Group: "coordination.k8s.io", Version: "v1alpha2", Kind: "LeaseCandidate"}: {"Age"}, + + {Group: "discovery.k8s.io", Version: "v1beta1", Kind: "EndpointSlice"}: {"Age"}, + + {Group: "flowcontrol.k8s.io", Version: "v1", Kind: "FlowSchema"}: {"Age"}, + {Group: "flowcontrol.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}: {"Age"}, + + {Group: "networking.k8s.io", Version: "v1beta1", Kind: "Ingress"}: {"Age"}, + {Group: "networking.k8s.io", Version: "v1beta1", Kind: "IngressClass"}: {"Age"}, + {Group: "networking.k8s.io", Version: "v1beta1", Kind: "IPAddress"}: {"Age"}, + {Group: "networking.k8s.io", Version: "v1beta1", Kind: "NetworkPolicy"}: {"Age"}, + {Group: "networking.k8s.io", Version: "v1beta1", Kind: "ServiceCIDR"}: {"Age"}, + + {Group: "node.k8s.io", Version: "v1", Kind: "RuntimeClass"}: {"Age"}, + + {Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"}: {"Age"}, + + {Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "ClusterRoleBinding"}: {"Age"}, + {Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "RoleBinding"}: {"Age"}, + + {Group: "resource.k8s.io", Version: "v1beta1", Kind: "DeviceClass"}: {"Age"}, + {Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaim"}: {"Age"}, + {Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaimTemplate"}: {"Age"}, + {Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceSlice"}: {"Age"}, + + {Group: "scheduling.k8s.io", Version: "v1", Kind: "PriorityClass"}: {"Age"}, + + {Group: "storage.k8s.io", Version: "v1", Kind: "CSIDriver"}: {"Age"}, + {Group: "storage.k8s.io", Version: "v1", Kind: "CSINode"}: {"Age"}, + {Group: "storage.k8s.io", Version: "v1", Kind: "StorageClass"}: {"Age"}, + {Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"}: {"Age"}, + {Group: "storage.k8s.io", Version: "v1beta1", Kind: "VolumeAttributesClass"}: {"Age"}, +} diff --git a/pkg/resources/common/util.go b/pkg/resources/common/util.go new file mode 100644 index 00000000..ddd3b8e2 --- /dev/null +++ b/pkg/resources/common/util.go @@ -0,0 +1,41 @@ +package common + +import ( + "strconv" + "strings" + + "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/steve/pkg/attributes" +) + +// GetIndexValueFromString looks for values between [ ]. +// e.g: $.metadata.fields[2], in this case it would return 2 +// In case it doesn't find any value between brackets it returns -1 +func GetIndexValueFromString(pathString string) int { + idxStart := strings.Index(pathString, "[") + if idxStart == -1 { + return -1 + } + idxEnd := strings.Index(pathString[idxStart+1:], "]") + if idxEnd == -1 { + return -1 + } + idx, err := strconv.Atoi(pathString[idxStart+1 : idxStart+1+idxEnd]) + if err != nil { + return -1 + } + return idx +} + +// GetColumnDefinitions returns ColumnDefinitions from an APISchema +func GetColumnDefinitions(schema *types.APISchema) []ColumnDefinition { + columns := attributes.Columns(schema) + if columns == nil { + return nil + } + colDefs, ok := columns.([]ColumnDefinition) + if !ok { + return nil + } + return colDefs +} diff --git a/pkg/resources/schema.go b/pkg/resources/schema.go index cb7e527c..e89c8305 100644 --- a/pkg/resources/schema.go +++ b/pkg/resources/schema.go @@ -47,9 +47,10 @@ func DefaultSchemaTemplates(cf *client.Factory, summaryCache *summarycache.SummaryCache, lookup accesscontrol.AccessSetLookup, discovery discovery.DiscoveryInterface, - namespaceCache corecontrollers.NamespaceCache) []schema.Template { + namespaceCache corecontrollers.NamespaceCache, + options common.TemplateOptions) []schema.Template { return []schema.Template{ - common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache), + common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, options), apigroups.Template(discovery), { ID: "configmap", @@ -77,10 +78,11 @@ func DefaultSchemaTemplatesForStore(store types.Store, baseSchemas *types.APISchemas, summaryCache *summarycache.SummaryCache, lookup accesscontrol.AccessSetLookup, - discovery discovery.DiscoveryInterface) []schema.Template { + discovery discovery.DiscoveryInterface, + options common.TemplateOptions) []schema.Template { return []schema.Template{ - common.DefaultTemplateForStore(store, summaryCache, lookup), + common.DefaultTemplateForStore(store, summaryCache, lookup, options), apigroups.Template(discovery), { ID: "configmap", diff --git a/pkg/resources/virtual/virtual.go b/pkg/resources/virtual/virtual.go index 105260c9..99fa4c0c 100644 --- a/pkg/resources/virtual/virtual.go +++ b/pkg/resources/virtual/virtual.go @@ -4,7 +4,10 @@ package virtual import ( "fmt" + "slices" + "time" + rescommon "github.com/rancher/steve/pkg/resources/common" "github.com/rancher/steve/pkg/resources/virtual/clusters" "github.com/rancher/steve/pkg/resources/virtual/common" "github.com/rancher/steve/pkg/resources/virtual/events" @@ -13,6 +16,8 @@ import ( "k8s.io/client-go/tools/cache" ) +var now = time.Now() + // TransformBuilder builds transform functions for specified GVKs through GetTransformFunc type TransformBuilder struct { defaultFields *common.DefaultFields @@ -28,13 +33,52 @@ func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder { } // GetTransformFunc returns the func to transform a raw object into a fixed object, if needed -func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc { +func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind, columns []rescommon.ColumnDefinition) cache.TransformFunc { converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0) if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" { converters = append(converters, events.TransformEventObject) } else if gvk.Kind == "Cluster" && gvk.Group == "management.cattle.io" && gvk.Version == "v3" { converters = append(converters, clusters.TransformManagedCluster) } + + // Detecting if we need to convert date fields + for _, col := range columns { + gvkDateFields, gvkFound := rescommon.DateFieldsByGVKBuiltins[gvk] + if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) { + converters = append(converters, func(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + index := rescommon.GetIndexValueFromString(col.Field) + if index == -1 { + return nil, fmt.Errorf("field index not found at column.Field struct variable: %s", col.Field) + } + + curValue, got, err := unstructured.NestedSlice(obj.Object, "metadata", "fields") + if !got { + return obj, nil + } + + if err != nil { + return nil, err + } + + value, cast := curValue[index].(string) + if !cast { + return nil, fmt.Errorf("could not cast metadata.fields %d to string", index) + } + duration, err := rescommon.ParseHumanReadableDuration(value) + if err != nil { + return nil, err + } + + curValue[index] = fmt.Sprintf("%d", now.Add(-duration).UnixMilli()) + if err := unstructured.SetNestedSlice(obj.Object, curValue, "metadata", "fields"); err != nil { + return nil, err + } + + return obj, nil + }) + } + } + converters = append(converters, t.defaultFields.TransformCommon) return func(raw interface{}) (interface{}, error) { diff --git a/pkg/resources/virtual/virtual_test.go b/pkg/resources/virtual/virtual_test.go index a0f31ce1..48ae070c 100644 --- a/pkg/resources/virtual/virtual_test.go +++ b/pkg/resources/virtual/virtual_test.go @@ -1,11 +1,12 @@ -package virtual_test +package virtual import ( "fmt" "strings" "testing" + "time" - "github.com/rancher/steve/pkg/resources/virtual" + rescommon "github.com/rancher/steve/pkg/resources/common" "github.com/rancher/steve/pkg/resources/virtual/common" "github.com/rancher/steve/pkg/summarycache" "github.com/rancher/wrangler/v3/pkg/summary" @@ -16,11 +17,14 @@ import ( ) func TestTransformChain(t *testing.T) { + now = func() time.Time { return time.Date(1992, 9, 2, 0, 0, 0, 0, time.UTC) }() + noColumns := []rescommon.ColumnDefinition{} tests := []struct { name string input any hasSummary *summary.SummarizedObject hasRelationships []summarycache.Relationship + columns []rescommon.ColumnDefinition wantOutput any wantError bool }{ @@ -65,6 +69,7 @@ func TestTransformChain(t *testing.T) { "id": "old-id", }, }, + columns: noColumns, wantOutput: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "test.cattle.io/v1", @@ -94,6 +99,131 @@ func TestTransformChain(t *testing.T) { }, }, }, + { + name: "CRD metadata.fields has a date field - should convert to timestamp", + hasSummary: &summary.SummarizedObject{ + PartialObjectMetadata: v1.PartialObjectMetadata{ + ObjectMeta: v1.ObjectMeta{ + Name: "testobj", + Namespace: "test-ns", + }, + TypeMeta: v1.TypeMeta{ + APIVersion: "test.cattle.io/v1", + Kind: "TestResource", + }, + }, + Summary: summary.Summary{ + State: "success", + Transitioning: false, + Error: false, + }, + }, + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "fields": []interface{}{"1d"}, + }, + "id": "old-id", + }, + }, + columns: []rescommon.ColumnDefinition{ + { + Field: "metadata.fields[0]", + TableColumnDefinition: v1.TableColumnDefinition{ + Name: "Age", + Type: "date", + }, + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "relationships": []any(nil), + "state": map[string]interface{}{ + "name": "success", + "error": false, + "transitioning": false, + "message": "", + }, + "fields": []interface{}{ + fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()), + }, + }, + "id": "test-ns/testobj", + "_id": "old-id", + }, + }, + }, + { + name: "built-in type metadata.fields has a date field - should convert to timestamp", + hasSummary: &summary.SummarizedObject{ + PartialObjectMetadata: v1.PartialObjectMetadata{ + ObjectMeta: v1.ObjectMeta{ + Name: "testobj", + Namespace: "test-ns", + }, + TypeMeta: v1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + Summary: summary.Summary{ + State: "success", + Transitioning: false, + Error: false, + }, + }, + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "fields": []interface{}{"1d"}, + }, + "id": "old-id", + }, + }, + columns: []rescommon.ColumnDefinition{ + { + TableColumnDefinition: v1.TableColumnDefinition{ + Name: "Age", + }, + Field: "metadata.fields[0]", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "relationships": []any(nil), + "state": map[string]interface{}{ + "name": "success", + "error": false, + "transitioning": false, + "message": "", + }, + "fields": []interface{}{ + fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()), + }, + }, + "id": "test-ns/testobj", + "_id": "old-id", + }, + }, + }, { name: "processable event", input: &unstructured.Unstructured{ @@ -118,6 +248,7 @@ func TestTransformChain(t *testing.T) { "type": "Gorniplatz", }, }, + columns: noColumns, wantOutput: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "/v1", @@ -162,6 +293,7 @@ func TestTransformChain(t *testing.T) { "type": "Gorniplatz", }, }, + columns: noColumns, wantOutput: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "palau.io/v1", @@ -218,6 +350,7 @@ func TestTransformChain(t *testing.T) { }, }, }, + columns: noColumns, wantOutput: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "management.cattle.io/v3", @@ -302,6 +435,7 @@ func TestTransformChain(t *testing.T) { }, }, }, + columns: noColumns, wantOutput: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "management.cattle.io/v3", @@ -346,13 +480,14 @@ func TestTransformChain(t *testing.T) { }, }, } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeCache := common.FakeSummaryCache{ SummarizedObject: test.hasSummary, Relationships: test.hasRelationships, } - tb := virtual.NewTransformBuilder(&fakeCache) + tb := NewTransformBuilder(&fakeCache) raw, isSignal, err := common.GetUnstructured(test.input) require.False(t, isSignal) require.Nil(t, err) @@ -362,7 +497,7 @@ func TestTransformChain(t *testing.T) { if test.name == "a non-ready cluster" { fmt.Printf("Stop here") } - output, err := tb.GetTransformFunc(gvk)(test.input) + output, err := tb.GetTransformFunc(gvk, test.columns)(test.input) require.Equal(t, test.wantOutput, output) if test.wantError { require.Error(t, err) diff --git a/pkg/server/server.go b/pkg/server/server.go index 830801de..162636af 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -224,7 +224,7 @@ func setup(ctx context.Context, server *Server) error { store := metricsStore.NewMetricsStore(errStore) // end store setup code - for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery()) { + for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), common.TemplateOptions{InSQLMode: true}) { sf.AddTemplate(template) } @@ -238,7 +238,7 @@ func setup(ctx context.Context, server *Server) error { return nil } } else { - for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache()) { + for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache(), common.TemplateOptions{InSQLMode: false}) { sf.AddTemplate(template) } onSchemasHandler = ccache.OnSchemas diff --git a/pkg/stores/sqlproxy/proxy_mocks_test.go b/pkg/stores/sqlproxy/proxy_mocks_test.go index e03065c9..909e8a05 100644 --- a/pkg/stores/sqlproxy/proxy_mocks_test.go +++ b/pkg/stores/sqlproxy/proxy_mocks_test.go @@ -14,6 +14,7 @@ import ( reflect "reflect" types "github.com/rancher/apiserver/pkg/types" + common "github.com/rancher/steve/pkg/resources/common" factory "github.com/rancher/steve/pkg/sqlcache/informer/factory" partition "github.com/rancher/steve/pkg/sqlcache/partition" sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes" @@ -389,15 +390,15 @@ func (m *MockTransformBuilder) EXPECT() *MockTransformBuilderMockRecorder { } // GetTransformFunc mocks base method. -func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind) cache.TransformFunc { +func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind, arg1 []common.ColumnDefinition) cache.TransformFunc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTransformFunc", arg0) + ret := m.ctrl.Call(m, "GetTransformFunc", arg0, arg1) ret0, _ := ret[0].(cache.TransformFunc) return ret0 } // GetTransformFunc indicates an expected call of GetTransformFunc. -func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0 any) *gomock.Call { +func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0, arg1) } diff --git a/pkg/stores/sqlproxy/proxy_store.go b/pkg/stores/sqlproxy/proxy_store.go index 7af34581..d642cfdc 100644 --- a/pkg/stores/sqlproxy/proxy_store.go +++ b/pkg/stores/sqlproxy/proxy_store.go @@ -240,7 +240,7 @@ type RelationshipNotifier interface { } type TransformBuilder interface { - GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc + GetTransformFunc(gvk schema.GroupVersionKind, colDefs []common.ColumnDefinition) cache.TransformFunc } type Store struct { @@ -335,9 +335,10 @@ func (s *Store) initializeNamespaceCache() error { // get any type-specific fields that steve is interested in fields = append(fields, getFieldForGVK(gvk)...) + cols := common.GetColumnDefinitions(&nsSchema) // get the type-specific transform func - transformFunc := s.transformBuilder.GetTransformFunc(gvk) + transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols) // get the ns informer tableClient := &tablelistconvert.Client{ResourceInterface: client} @@ -545,7 +546,7 @@ func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types. gvk := attributes.GVK(schema) fields := getFieldsFromSchema(schema) fields = append(fields, getFieldForGVK(gvk)...) - transformFunc := s.transformBuilder.GetTransformFunc(gvk) + transformFunc := s.transformBuilder.GetTransformFunc(gvk, nil) tableClient := &tablelistconvert.Client{ResourceInterface: client} attrs := attributes.GVK(schema) ns := attributes.Namespaced(schema) @@ -756,7 +757,9 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, apiSchema *types.APISc gvk := attributes.GVK(apiSchema) fields := getFieldsFromSchema(apiSchema) fields = append(fields, getFieldForGVK(gvk)...) - transformFunc := s.transformBuilder.GetTransformFunc(gvk) + cols := common.GetColumnDefinitions(apiSchema) + + transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols) tableClient := &tablelistconvert.Client{ResourceInterface: client} attrs := attributes.GVK(apiSchema) ns := attributes.Namespaced(apiSchema) diff --git a/pkg/stores/sqlproxy/proxy_store_test.go b/pkg/stores/sqlproxy/proxy_store_test.go index 7507604f..ed40875a 100644 --- a/pkg/stores/sqlproxy/proxy_store_test.go +++ b/pkg/stores/sqlproxy/proxy_store_test.go @@ -245,7 +245,7 @@ func TestListByPartitions(t *testing.T) { cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) - tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) assert.Nil(t, err) @@ -462,7 +462,7 @@ func TestListByPartitions(t *testing.T) { // note also the watchable bool is expected to be false cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil) - tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) assert.Nil(t, err) @@ -534,7 +534,7 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error")) _, _, _, err = s.ListByPartitions(req, schema, partitions) @@ -613,7 +613,7 @@ func TestListByPartitions(t *testing.T) { // This tests that fields are being extracted from schema columns and the type specific fields map cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error")) - tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) _, _, _, err = s.ListByPartitions(req, schema, partitions) assert.NotNil(t, err) @@ -725,7 +725,7 @@ func TestListByPartitionWithUserAccess(t *testing.T) { attributes.SetGVK(theSchema, gvk) cg.EXPECT().TableAdminClient(apiOp, theSchema, "", &WarningBuffer{}).Return(ri, nil) cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {"id"}, {"metadata", "state", "name"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(theSchema), attributes.Namespaced(theSchema), true).Return(c, nil) - tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) listToReturn := &unstructured.UnstructuredList{ Items: make([]unstructured.Unstructured, 0, 0), @@ -767,7 +767,7 @@ func TestReset(t *testing.T) { cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil) - tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.Nil(t, err) assert.Equal(t, nsc2, s.namespaceCache) @@ -874,7 +874,7 @@ func TestReset(t *testing.T) { cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error")) - tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.NotNil(t, err) },